Kaggle competition: Mercari Price Suggestion Challenge
X_train = df_work_sprs[:y_train.shape[0]]
X_test = df_work_sprs[y_train.shape[0]:]<compute_train_metric>
from sklearn.linear_model import Ridge
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import OneHotEncoder
from scipy.sparse import hstack
%%time
C = 0.12
clf = LogisticRegression(C=C, solver='lbfgs', max_iter=1000, verbose=0, n_jobs=-1)
 score = cross_validate(clf, X_train, y_train, cv=3, scoring="roc_auc")
 mean = score['test_score'].mean()
print(score['test_score'])
 print('C =', C, f'{mean:.8f}' )<train_model>
train_df = pd.read_csv('.. /input/train.tsv', delimiter='\t')
 test_df = pd.read_csv('.. /input/test.tsv', delimiter='\t' )
%%time
clf = LogisticRegression(C=C, solver='lbfgs', max_iter=1000, verbose=0, n_jobs=-1)
 clf.fit(X_train, y_train )<predict_on_test>
train_test_border = len(train_df )
y_preds = clf.predict_proba(X_test)[:,1]<save_to_csv>
train_test_df = pd.concat([train_df.drop(['train_id', 'price'], axis=1), test_df.drop(['test_id'], axis=1)], axis=0 )
sample_submission['target'] = y_preds
sample_submission.to_csv('submission.csv' )<load_from_csv>
train_test_df.isnull().sum()
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv', index_col='id')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv', index_col='id' )<drop_column>
train_test_df.category_name = train_test_df.category_name.fillna('Nan/Nan/Nan')
 train_test_df.brand_name = train_test_df.brand_name.fillna('Nan')
 train_test_df.item_description = train_test_df.item_description.fillna('No description yet' )
train['ord_5_1'] = train['ord_5'].str[0]
train['ord_5_2'] = train['ord_5'].str[1]
train = train.drop('ord_5', axis=1)
 test['ord_5_1'] = test['ord_5'].str[0]
test['ord_5_2'] = test['ord_5'].str[1]
test = test.drop('ord_5', axis=1 )<feature_engineering>
train_test_df.isnull().sum()
columns_to_test = ['nom_7', 'nom_8', 'nom_9']
replace_xor = lambda x: 'xor' if x in xor_values else x
for column in columns_to_test:
xor_values = set(train[column].unique())^ set(test[column].unique())
 if xor_values:
print('Column', column, 'has', len(xor_values), 'XOR values')
 train[column] = train[column].apply(replace_xor)
 test[column] = test[column].apply(replace_xor)
 else:
print('Column', column, 'has no XOR values' )<prepare_x_and_y>
split_category = np.array([x for x in train_test_df.category_name.str.split('/', 2 ).values] )
y_train = train['target'].copy()
x_train = train.drop('target', axis=1)
 del train
x_test = test.copy()
del test<categorify>
train_test_df['split_category_0'] = split_category[:,0]
train_test_df['split_category_1'] = split_category[:,1]
train_test_df['split_category_2'] = split_category[:,2]
train_test_df['brand_category_2'] = train_test_df.split_category_2.values + '/' + train_test_df.brand_name.values
train_part = len(x_train)
 traintest = pd.concat([x_train, x_test])
 df = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True)
 x_train = df[:train_part]
x_test = df[train_part:]
del df
del traintest<concatenate>
train_test_df = train_test_df.drop(['category_name'], axis=1 )
x_train = x_train.sparse.to_coo().tocsr()
x_test = x_test.sparse.to_coo().tocsr()<import_modules>
train_test_df.split_category_0.value_counts()
from sklearn.model_selection import KFold, StratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_auc_score
from cross_validation_framework import *<train_on_grid>
category_df = train_test_df[train_test_df.split_category_0 == 'Women']
logit_param_grid = {
'C': [0.100, 0.150, 0.120, 0.125, 0.130, 0.135, 0.140, 0.145, 0.150]
}
logit_grid = GridSearchCV(LogisticRegression(solver='lbfgs'), logit_param_grid,
scoring='roc_auc', cv=5, n_jobs=-1, verbose=0)
 logit_grid.fit(x_train, y_train)
 best_C = logit_grid.best_params_['C']
print('Best C:', best_C )<predict_on_test>
name = category_df.name.values
item_desc = category_df.item_description.values
logit = LogisticRegression(C=best_C, solver='lbfgs', class_weight='balanced', max_iter=10000)
 cv = KFold(n_splits=10, random_state=42)
 oof, trained_estimators = fit(ScikitLearnPredictProbaEstimator(logit), roc_auc_score, x_train, y_train, cv)
 y = predict(trained_estimators, x_test )<save_to_csv>
vectorizer = TfidfVectorizer(max_df=0.85, min_df=2, ngram_range=(1,2), max_features=100000)
 name_vecs = vectorizer.fit_transform(name)
 item_desc_vecs = vectorizer.fit_transform(item_desc )
submission = pd.read_csv('.. /input/cat-in-the-dat/sample_submission.csv', index_col='id')
 submission['target'] = y
submission.to_csv('logit.csv' )<set_options>
category_df = category_df.drop(['name', 'item_description', 'split_category_0'], axis=1 )
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 warnings.simplefilter('ignore')
 sns.set()
%matplotlib inline 
<categorify>
category_df.split_category_1 = category_df.split_category_1.astype('category')
 category_df.split_category_2 = category_df.split_category_2.astype('category')
 category_df.brand_name = category_df.brand_name.astype('category')
 category_df.brand_category_2 = category_df.brand_category_2.astype('category')
 category_df.split_category_1 = category_df.split_category_1.cat.codes
category_df.split_category_2 = category_df.split_category_2.cat.codes
category_df.brand_name = category_df.brand_name.cat.codes
category_df.brand_category_2 = category_df.brand_category_2.cat.codes
def encode_FE(df1, df2, cols):
for col in cols:
df = pd.concat([df1[col],df2[col]])
 vc = df.value_counts(dropna=True, normalize=True ).to_dict()
vc[-1] = -1
nm = col+'_FE'
df1[nm] = df1[col].map(vc)
 df1[nm] = df1[nm].astype('float32')
 df2[nm] = df2[col].map(vc)
 df2[nm] = df2[nm].astype('float32')
 print(nm,', ',end='')
 def encode_LE(col,train,test,verbose=False):
df_comb = pd.concat([train[col],test[col]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 nm = col
if df_comb.max() >32000:
train[nm] = df_comb[:len(train)].astype('int32')
 test[nm] = df_comb[len(train):].astype('int32')
 else:
train[nm] = df_comb[:len(train)].astype('int16')
 test[nm] = df_comb[len(train):].astype('int16')
 del df_comb; x=gc.collect()
if verbose: print(nm,', ',end='')
 def encode_AG(main_columns, uids, aggregations, train_df, test_df,
fillna=True, usena=False):
for main_column in main_columns:
for col in uids:
for agg_type in aggregations:
new_col_name = main_column+'_'+col+'_'+agg_type
temp_df = pd.concat([train_df[[col, main_column]], test_df[[col,main_column]]])
 if usena: temp_df.loc[temp_df[main_column]==-1,main_column] = np.nan
temp_df = temp_df.groupby([col])[main_column].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train_df[new_col_name] = train_df[col].map(temp_df ).astype('float32')
 test_df[new_col_name] = test_df[col].map(temp_df ).astype('float32')
 if fillna:
train_df[new_col_name].fillna(-1,inplace=True)
 test_df[new_col_name].fillna(-1,inplace=True)
 print("'"+new_col_name+"'",', ',end='')
 def encode_CB(col1,col2,train,test):
nm = col1+'_'+col2
train[nm] = train[col1].astype(str)+'_'+train[col2].astype(str)
 test[nm] = test[col1].astype(str)+'_'+test[col2].astype(str)
 encode_LE(nm,train,test)
 def encode_AG2(main_columns, uids, train_df, test_df):
for main_column in main_columns:
for col in uids:
comb = pd.concat([train_df[[col]+[main_column]],test_df[[col]+[main_column]]],axis=0)
 mp = comb.groupby(col)[main_column].agg(['nunique'])['nunique'].to_dict()
train_df[col+'_'+main_column+'_ct'] = train_df[col].map(mp ).astype('float32')
 test_df[col+'_'+main_column+'_ct'] = test_df[col].map(mp ).astype('float32')
 print(col+'_'+main_column+'_ct, ',end='' )<feature_engineering>
encoder = OneHotEncoder()
def comb_mails(emails,us_emails,train,test):
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )<categorify>
X_data = encoder.fit_transform(category_df )
def love() :
print('
'.join([''.join([(' I_Love_Data_Science_'[(x-y)% len('I_Love_Data_Science_')] if(( x*0.05)**2+(y*0.1)**2-1)**3-(x*0.05)**2*(y*0.1)**3 <= 0 else ' ')for x in range(-30, 30)])for y in range(15, -15, -1)]))<load_from_csv>
X_data = hstack([X_data, name_vecs, item_desc_vecs] ).tocsr()
def combine() :
with timer('Combining :'):
print('Combining Start...')
 train_id= pd.read_csv("/kaggle/input/ieee-fraud-detection/train_identity.csv")
 test_id = pd.read_csv("/kaggle/input/ieee-fraud-detection/test_identity.csv")
 train_tr = pd.read_csv("/kaggle/input/ieee-fraud-detection/train_transaction.csv")
 test_tr = pd.read_csv("/kaggle/input/ieee-fraud-detection/test_transaction.csv")
 train=pd.merge(train_tr, train_id, on = "TransactionID",how='left',left_index=True, right_index=True)
 test=pd.merge(test_tr, test_id, on = "TransactionID",how="left",left_index=True, right_index=True)
 del train_id, train_tr, test_id, test_tr
test.columns=train.columns.drop("isFraud")
 return train,test<categorify>
train_df.category_name = train_df.category_name.fillna('Nan/Nan/Nan')
 train_df['split_category_0'] = np.array([x for x in train_df.category_name.str.split('/', 2 ).values])[:,0]
train_df.split_category_0.value_counts()
def pre_processing_and_feature_engineering() :
train,test=combine()
with timer('Preprocessing and Feature Engineering'):
print('-' * 30)
 print('Preprocessing and Feature Engineering start...')
 print('-' * 10)
 print("After corelation test we can drop this columns")
 drop_col=[]
train_colmns=train.loc[:,"V1":"V339"].columns
test_columns=test.loc[:,"V2":"V339"].columns
for col1,col2 in zip(train_colmns,test_columns):
if(( train.loc[:,col1:col2].corr().loc[col2].sum() -1)>0.75)&(train[col1].isna().sum() == train[col2].isna().sum()):
print("'"+col2+"'",', ',end='')
 drop_col.append(col2)
 train=train.drop(drop_col,axis=1)
 test=test.drop(drop_col,axis=1)
 del drop_col
print("-")
 print("We drop these columns as well because more than 90 percent of these columns are nan")
 for col in train.columns:
if sum(train[col].isnull())/float(len(train.index)) > 0.90:
print("'"+col+"'",', ',end='')
 train=train.drop(col,axis=1)
 test=test.drop(col,axis=1)
 emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum',
'scranton.edu': 'other', 'optonline.net': 'other', 'hotmail.co.uk': 'microsoft',
'comcast.net': 'other', 'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo', 'charter.net': 'spectrum', 'live.com': 'microsoft',
'aim.com': 'aol', 'hotmail.de': 'microsoft', 'centurylink.net': 'centurylink',
'gmail.com': 'google', 'me.com': 'apple', 'earthlink.net': 'other', 'gmx.de': 'other',
'web.de': 'other', 'cfl.rr.com': 'other', 'hotmail.com': 'microsoft',
'protonmail.com': 'other', 'hotmail.fr': 'microsoft', 'windstream.net': 'other',
'outlook.es': 'microsoft', 'yahoo.co.jp': 'yahoo', 'yahoo.de': 'yahoo',
'servicios-ta.com': 'other', 'netzero.net': 'other', 'suddenlink.net': 'other',
'roadrunner.com': 'other', 'sc.rr.com': 'other', 'live.fr': 'microsoft',
'verizon.net': 'yahoo', 'msn.com': 'microsoft', 'q.com': 'centurylink',
'prodigy.net.mx': 'att', 'frontier.com': 'yahoo', 'anonymous.com': 'other',
'rocketmail.com': 'yahoo', 'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo', 'outlook.com': 'microsoft', 'mail.com': 'other',
'bellsouth.net': 'other', 'embarqmail.com': 'centurylink', 'cableone.net': 'other',
'hotmail.es': 'microsoft', 'mac.com': 'apple', 'yahoo.co.uk': 'yahoo', 'netzero.com': 'other',
'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other', 'cox.net': 'other',
'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']
comb_mails(emails,us_emails,train,test)
 print("-")
 print("Creating New Features...")
 na_low=['TransactionAmt' , 'ProductCD' , 'card1' , 'card2' , 'card3' , 'card4' , 'card5' , 'card6' , 'addr1' ,
'addr2' , 'P_emaildomain' , 'C1' , 'C2' , 'C3' , 'C4' , 'C5' , 'C6' , 'C7' , 'C8' , 'C9' , 'C10' ,
'C11' , 'C12' , 'C13' , 'C14' , 'M6' ]
numeric=train[na_low]._get_numeric_data().columns
encode_FE(train,test,['addr1','card1','card2','card3','P_emaildomain'])
 encode_CB('card1','addr1',train,test)
 train['day'] = train.TransactionDT /(24*60*60)
 train['uid'] = train.card1_addr1.astype(str)+'_'+np.floor(train.day-train.D1 ).astype(str)
 test['day'] = test.TransactionDT /(24*60*60)
 test['uid'] = test.card1_addr1.astype(str)+'_'+np.floor(test.day-test.D1 ).astype(str)
 encode_FE(train,test,['uid'])
 encode_AG(numeric, ['uid'],['mean',"std"], train, test)
 categorical_columns=test.columns.drop(test._get_numeric_data().columns)
 categorical_columns=categorical_columns.drop('uid')
 encode_AG2(categorical_columns, ['uid'], train, test)
 encode_FE(train,test,['addr1','card1','card2','card3','P_emaildomain'])
 encode_CB('card1_addr1','P_emaildomain',train, test)
 encode_FE(train,test,['card1_addr1','card1_addr1_P_emaildomain'])
 encode_AG(['TransactionAmt','D9','D11'],['card1','card1_addr1','card1_addr1_P_emaildomain'],['mean','std'],train,test,usena=True)
 del train['uid'], test['uid']
print("Creating New Features Finished")
 print('-' * 10)
 print('Label Coding and One Hot Encoding Start...')
 categorical_columns=test.columns.drop(test._get_numeric_data().columns)
 for i in categorical_columns:
lbe=preprocessing.LabelEncoder()
train[i]=lbe.fit_transform(train[i].astype(str))
for i in categorical_columns:
test[i]=lbe.fit_transform(test[i].astype(str))
for i in categorical_columns:
if(test[i].max() == train[i].max())&(train[i].max() <8):
test = pd.get_dummies(test, columns = [i])
 train=pd.get_dummies(train, columns = [i])
 print('-' * 10)
 return train,test<prepare_x_and_y>
X_data = np.log1p(X_data)
 y_train = np.log1p(train_df[train_df.split_category_0 == 'Women'].price )
def modeling() :
train,test=pre_processing_and_feature_engineering()
with timer('Machine Learning '):
train_TransactionID= train["TransactionID"]
test_TransactionID=test["TransactionID"]
X= train.sort_values('TransactionDT' ).drop([ 'TransactionDT', 'TransactionID'], axis=1)
 y = train.sort_values('TransactionDT')['isFraud']
test = test.sort_values('TransactionDT' ).drop(['TransactionDT', 'TransactionID'], axis=1)
 del train
X=X.drop("isFraud", axis=1)
 print('-' * 30)
 print("Droping unusefull Features which were determined end of the first training...")
 print('-')
 print('-' * 20)
 print('Machine Learning start...')
 params = {'num_leaves': 491,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47}
folds = TimeSeriesSplit(n_splits=5)
 aucs = list()
feature_importances = pd.DataFrame()
feature_importances['feature'] = X.columns
for fold,(trn_idx, test_idx)in enumerate(folds.split(X, y)) :
print('Training on fold {}'.format(fold + 1))
trn_data = lgb.Dataset(X.iloc[trn_idx], label=y.iloc[trn_idx])
 val_data = lgb.Dataset(X.iloc[test_idx], label=y.iloc[test_idx])
 clf = lgb.train(params, trn_data, 10000, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds=500)
 feature_importances['fold_{}'.format(fold + 1)] = clf.feature_importance()
aucs.append(clf.best_score['valid_1']['auc'])
 print('Fold {} finished '.format(fold + 1))
print('-' * 10)
 print('Training has finished.')
 print('Mean AUC:', np.mean(aucs))
print('-' * 10)
 print("final model, set the output as a dataframe and convert to csv file named submission.csv")
 best_iter = clf.best_iteration
clf = lgb.LGBMClassifier(**params, num_boost_round=best_iter)
 clf.fit(X, y)
 predictions = clf.predict_proba(test)[:, 1]
output = pd.DataFrame({ "TransactionID" : test_TransactionID, "isFraud": predictions })
 output.to_csv('submission_lgbm.csv', index=False)
 print('Feature importances...')
 feature_importances['average'] = feature_importances[['fold_{}'.format(fold + 1)for fold in range(folds.n_splits)]].mean(axis=1)
 feature_importances.to_csv('feature_importances.csv')
 love()
plt.figure(figsize=(16, 16))
sns.barplot(data=feature_importances.sort_values(by='average', ascending=False ).head(50), x='average', y='feature');
plt.title('50 TOP feature importance ');
return<train_model>
category_index = category_df.index
def main() :
with timer('Full Model Run '):
print("Full Model Run Start...")
 print('-' * 50)
 modeling()
print('-' * 50 )<set_options>
train_women = train_df.split_category_0.value_counts() ['Women']
warnings.filterwarnings('ignore' )<init_hyperparams>
submit_index = category_index[train_women:]
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df<define_variables>
X_train = X_data[:train_women]
X_submit = X_data[train_women:]
SEED = 42
seed_everything(SEED)
 LOCAL_TEST = False
TARGET = 'isFraud'
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d' )<load_pretrained>
ridge = Ridge(alpha=3.1 )
print('Load Data')
 train_df = pd.read_pickle('.. /input/mydata/train_df_with_mn.pkl')
 test_df = pd.read_pickle('.. /input/mydata/test_df_with_mn.pkl')
 remove_features = pd.read_pickle('.. /input/mydata/remove_features_with_mn.pkl')
<categorify>
ridge.fit(X_train, y_train )
features_columns = [col for col in list(train_df)if col not in remove_features]
if not LOCAL_TEST:
train_df = reduce_mem_usage(train_df)
 test_df = reduce_mem_usage(test_df )<data_type_conversions>
price = np.expm1(ridge.predict(X_submit))
train_df[['P_emaildomain', 'R_emaildomain', 'DeviceType']] = train_df[['P_emaildomain', 'R_emaildomain', 'DeviceType']].astype(int)
 test_df[['P_emaildomain', 'R_emaildomain', 'DeviceType']] = train_df[['P_emaildomain', 'R_emaildomain', 'DeviceType']].astype(int )<prepare_x_and_y>
submit_df = pd.DataFrame(submit_index)
 submit_df = pd.concat([submit_df, pd.DataFrame(price)], axis=1)
 submit_df.columns = ['test_id', 'price']
submit_df.head()
X_train = train_df[features_columns]
y_train = train_df[TARGET]
X_test = test_df[features_columns]<prepare_x_and_y>
category_df = train_test_df[train_test_df.split_category_0 == 'Beauty']
NFOLDS = 6
folds = GroupKFold(n_splits = NFOLDS)
 y_preds = np.zeros(X_test.shape[0])
 split_groups = train_df['DT_M']<drop_column>
name = category_df.name.values
item_desc = category_df.item_description.values
del train_df, test_df<split>
vectorizer = TfidfVectorizer(max_df=0.85, min_df=2, ngram_range=(1,2), max_features=100000)
 name_vecs = vectorizer.fit_transform(name)
 item_desc_vecs = vectorizer.fit_transform(item_desc )
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train, y_train, groups=split_groups)) :
clf = xgb.XGBClassifier(
 n_estimators=512,
max_depth=16,
learning_rate=0.014,
subsample=0.85,
colsample_bytree=0.85,
tree_method='gpu_hist',
reg_alpha=0.3,
reg_lamdba=0.243 
)
 X_tr, X_vl = X_train.iloc[trn_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[trn_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
print('ROC AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 del X_tr, X_vl, y_tr, y_vl
y_preds[:int(len(X_test)/2)] += clf.predict_proba(X_test[:int(len(X_test)/2)])[:,1] / NFOLDS
y_preds[int(len(X_test)/2):] += clf.predict_proba(X_test[int(len(X_test)/2):])[:,1] / NFOLDS
gc.collect()<save_to_csv>
category_df = category_df.drop(['name', 'item_description', 'split_category_0'], axis=1 )
if not LOCAL_TEST:
submission = pd.read_csv(".. /input/ieee-fraud-detection/sample_submission.csv")
 submission['isFraud'] = y_preds
submission.to_csv('submission_xgb_groupkfold_final.csv', index=False )<install_modules>
category_df.split_category_1 = category_df.split_category_1.astype('category')
 category_df.split_category_2 = category_df.split_category_2.astype('category')
 category_df.brand_name = category_df.brand_name.astype('category')
 category_df.brand_category_2 = category_df.brand_category_2.astype('category')
 category_df.split_category_1 = category_df.split_category_1.cat.codes
category_df.split_category_2 = category_df.split_category_2.cat.codes
category_df.brand_name = category_df.brand_name.cat.codes
category_df.brand_category_2 = category_df.brand_category_2.cat.codes
!pip install missingpy<install_modules>
encoder = OneHotEncoder()
!pip install git+https://github.com/t0biK/data_science_utils.git<set_options>
X_data = encoder.fit_transform(category_df )
%matplotlib inline 
<load_from_csv>
X_data = hstack([X_data, name_vecs, item_desc_vecs] ).tocsr()
test_identity = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_identity.csv')
 test_identity.columns = test_identity.columns.str.replace("-", "_")
 train_identity = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_identity.csv')
 test_transaction = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_transaction.csv')
 train_transaction = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_transaction.csv' )<merge>
X_data = np.log1p(X_data)
 y_train = np.log1p(train_df[train_df.split_category_0 == 'Beauty'].price )
train = pd.merge(train_identity, train_transaction, on='TransactionID', how='right')
 test = pd.merge(test_identity, test_transaction, on='TransactionID', how='right' )<feature_engineering>
category_index = category_df.index
test_identity, train_identity, test_transaction, train_transaction = [None]*4<define_variables>
train_category = train_df.split_category_0.value_counts() ['Beauty']
numerical_columns = ['id_0'+str(i)for i in range(1,10)]+['id_10', 'id_11']<feature_engineering>
submit_index = category_index[train_category:]
test.loc[:,~test.columns.isin(numerical_columns)] = test.loc[:,~test.columns.isin(numerical_columns)].fillna(-999)
 train.loc[:,~train.columns.isin(numerical_columns)] = train.loc[:,~train.columns.isin(numerical_columns)].fillna(-999 )<categorify>
X_train = X_data[:train_category]
X_submit = X_data[train_category:]
imputer = MissForest()
test.loc[:,test.columns.isin(numerical_columns)] = imputer.fit_transform(test.loc[:,test.columns.isin(numerical_columns)])
 train.loc[:,train.columns.isin(numerical_columns)] = imputer.fit_transform(train.loc[:,train.columns.isin(numerical_columns)] )<feature_engineering>
ridge = Ridge(alpha=3.1)
 ridge.fit(X_train, y_train )
params = {'objective': 'binary', "boosting_type": "gbdt", "subsample": 1, "bagging_seed": 11, "metric": 'auc', 'num_boost_round':100, 'verbose':-1} 
<define_variables>
price = np.expm1(ridge.predict(X_submit))
useful_features = ['TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13',
'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M2', 'M3',
'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V17',
'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48',
'V49', 'V51', 'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V69', 'V70', 'V71',
'V72', 'V73', 'V74', 'V75', 'V76', 'V78', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128', 'V130', 'V131', 'V138', 'V139', 'V140',
'V143', 'V145', 'V146', 'V147', 'V149', 'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161',
'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173', 'V175', 'V176', 'V177',
'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189', 'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204',
'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219', 'V220',
'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V231', 'V233', 'V234', 'V238', 'V239',
'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261',
'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276',
'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285', 'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303',
'V304', 'V306', 'V307', 'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324', 'V326',
'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338', 'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09',
'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33',
'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo']<data_type_conversions>
tmp_df = pd.DataFrame(submit_index)
 tmp_df = pd.concat([tmp_df, pd.DataFrame(price)], axis=1)
 tmp_df.columns = ['test_id', 'price']
submit_df = pd.concat([submit_df, tmp_df], axis=0)
 submit_df
train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 train['card1_count_full'] = train['card1'].map(pd.concat([train['card1'], test['card1']], ignore_index=True ).value_counts(dropna=False))
test['card1_count_full'] = test['card1'].map(pd.concat([train['card1'], test['card1']], ignore_index=True ).value_counts(dropna=False))
train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24
for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
for feature in ['id_34', 'id_36']:
if feature in useful_features:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_35', 'id_36']:
if feature in useful_features:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))<init_hyperparams>
category_df = train_test_df[train_test_df.split_category_0 == 'Kids']
category_df.head()
params = {'num_leaves': 491,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.0066,
'n_estimators': 1000,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}<split>
name = category_df.name.values
item_desc = category_df.item_description.values
vectorizer = TfidfVectorizer(max_df=0.85, min_df=2, ngram_range=(1,2), max_features=100000)
 name_vecs = vectorizer.fit_transform(name)
 item_desc_vecs = vectorizer.fit_transform(item_desc )
X_train, X_test, y_train, y_test = train_test_split(train.drop(columns=['isFraud', 'TransactionID', 'TransactionDT']), train['isFraud'], test_size=0.33 )<data_type_conversions>
category_df = category_df.drop(['name', 'item_description', 'split_category_0'], axis=1)
 category_df.split_category_1 = category_df.split_category_1.astype('category')
 category_df.split_category_2 = category_df.split_category_2.astype('category')
 category_df.brand_name = category_df.brand_name.astype('category')
 category_df.brand_category_2 = category_df.brand_category_2.astype('category')
 category_df.split_category_1 = category_df.split_category_1.cat.codes
category_df.split_category_2 = category_df.split_category_2.cat.codes
category_df.brand_name = category_df.brand_name.cat.codes
category_df.brand_category_2 = category_df.brand_category_2.cat.codes
for col in X_train.select_dtypes(include='object' ).columns:
le = LabelEncoder()
le.fit(list(X_train[col].astype(str ).values)+ list(X_test[col].astype(str ).values)+ list(test[col].astype(str ).values))
X_train[col] = le.transform(list(X_train[col].astype(str ).values))
X_test[col] = le.transform(list(X_test[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))<train_model>
encoder = OneHotEncoder()
X_data = encoder.fit_transform(category_df)
 X_data = hstack([X_data, name_vecs, item_desc_vecs] ).tocsr()
X_data = np.log1p(X_data)
 y_train = np.log1p(train_df[train_df.split_category_0 == 'Kids'].price)
 category_index = category_df.index
train_category = train_df.split_category_0.value_counts() ['Kids']
submit_index = category_index[train_category:]
X_train = X_data[:train_category]
X_submit = X_data[train_category:]
clf = lgb.LGBMClassifier(**params)
 clf.fit(X_train, y_train )<define_variables>
ridge = Ridge(alpha=3.1)
 ridge.fit(X_train, y_train )
cols_to_drop = ['TransactionID', 'TransactionDT']<save_to_csv>
price = np.expm1(ridge.predict(X_submit))
sub = pd.read_csv('/kaggle/input/ieee-fraud-detection/sample_submission.csv')
 sub['isFraud'] = clf.predict_proba(test.drop(columns=cols_to_drop)) [:, 1]
sub['TransactionID'] = test.TransactionID
sub.to_csv('submission.csv', index=False )<import_modules>
tmp_df = pd.DataFrame(submit_index)
 tmp_df = pd.concat([tmp_df, pd.DataFrame(price)], axis=1)
 tmp_df.columns = ['test_id', 'price']
submit_df = pd.concat([submit_df, tmp_df], axis=0)
 submit_df
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
 %matplotlib inline
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df<set_options>
category_df = train_test_df[train_test_df.split_category_0 == 'Electronics']
category_df.head()
gc.collect()<load_from_csv>
name = category_df.name.values
item_desc = category_df.item_description.values
vectorizer = TfidfVectorizer(max_df=0.85, min_df=2, ngram_range=(1,2), max_features=100000)
 name_vecs = vectorizer.fit_transform(name)
 item_desc_vecs = vectorizer.fit_transform(item_desc )
%%time
train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv')
 train_identity = pd.read_csv(f'{folder_path}train_identity.csv')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv')
 sample_submission = pd.read_csv(f'{folder_path}sample_submission.csv', index_col='TransactionID')
 df_train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True, on='TransactionID')
 df_test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True, on='TransactionID')
 print(df_train.shape)
 print(df_test.shape)
 del train_transaction, train_identity, test_transaction, test_identity<drop_column>
category_df = category_df.drop(['name', 'item_description', 'split_category_0'], axis=1)
 category_df.split_category_1 = category_df.split_category_1.astype('category')
 category_df.split_category_2 = category_df.split_category_2.astype('category')
 category_df.brand_name = category_df.brand_name.astype('category')
 category_df.brand_category_2 = category_df.brand_category_2.astype('category')
 category_df.split_category_1 = category_df.split_category_1.cat.codes
category_df.split_category_2 = category_df.split_category_2.cat.codes
category_df.brand_name = category_df.brand_name.cat.codes
category_df.brand_category_2 = category_df.brand_category_2.cat.codes
%%time
df_train = reduce_mem_usage(df_train)
 df_test = reduce_mem_usage(df_test )<define_variables>
encoder = OneHotEncoder()
X_data = encoder.fit_transform(category_df)
 X_data = hstack([X_data, name_vecs, item_desc_vecs] ).tocsr()
X_data = np.log1p(X_data)
 y_train = np.log1p(train_df[train_df.split_category_0 == 'Electronics'].price)
 category_index = category_df.index
train_category = train_df.split_category_0.value_counts() ['Electronics']
submit_index = category_index[train_category:]
X_train = X_data[:train_category]
X_submit = X_data[train_category:]
%%time
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum',
'scranton.edu': 'other', 'optonline.net': 'other', 'hotmail.co.uk': 'microsoft',
'comcast.net': 'other', 'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo', 'charter.net': 'spectrum', 'live.com': 'microsoft',
'aim.com': 'aol', 'hotmail.de': 'microsoft', 'centurylink.net': 'centurylink',
'gmail.com': 'google', 'me.com': 'apple', 'earthlink.net': 'other', 'gmx.de': 'other',
'web.de': 'other', 'cfl.rr.com': 'other', 'hotmail.com': 'microsoft',
'protonmail.com': 'other', 'hotmail.fr': 'microsoft', 'windstream.net': 'other',
'outlook.es': 'microsoft', 'yahoo.co.jp': 'yahoo', 'yahoo.de': 'yahoo',
'servicios-ta.com': 'other', 'netzero.net': 'other', 'suddenlink.net': 'other',
'roadrunner.com': 'other', 'sc.rr.com': 'other', 'live.fr': 'microsoft',
'verizon.net': 'yahoo', 'msn.com': 'microsoft', 'q.com': 'centurylink',
'prodigy.net.mx': 'att', 'frontier.com': 'yahoo', 'anonymous.com': 'other',
'rocketmail.com': 'yahoo', 'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo', 'outlook.com': 'microsoft', 'mail.com': 'other',
'bellsouth.net': 'other', 'embarqmail.com': 'centurylink', 'cableone.net': 'other',
'hotmail.es': 'microsoft', 'mac.com': 'apple', 'yahoo.co.uk': 'yahoo', 'netzero.com': 'other',
'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other', 'cox.net': 'other',
'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']
for c in ['P_emaildomain', 'R_emaildomain']:
df_train[c + '_bin'] = df_train[c].map(emails)
 df_test[c + '_bin'] = df_test[c].map(emails)
 df_train[c + '_suffix'] = df_train[c].map(lambda x: str(x ).split('.')[-1])
 df_test[c + '_suffix'] = df_test[c].map(lambda x: str(x ).split('.')[-1])
 df_train[c + '_suffix'] = df_train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 df_test[c + '_suffix'] = df_test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )<categorify>
ridge = Ridge(alpha=3.1)
 ridge.fit(X_train, y_train )
%%time
for f in df_train.drop('isFraud', axis=1 ).columns:
if df_train[f].dtype=='object' or df_test[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(df_train[f].values)+ list(df_test[f].values))
df_train[f] = lbl.transform(list(df_train[f].values))
df_test[f] = lbl.transform(list(df_test[f].values))<categorify>
price = np.expm1(ridge.predict(X_submit))
%%time
df_train = df_train.fillna(-999)
 df_test = df_test.fillna(-999 )<prepare_x_and_y>
tmp_df = pd.DataFrame(submit_index)
 tmp_df = pd.concat([tmp_df, pd.DataFrame(price)], axis=1)
 tmp_df.columns = ['test_id', 'price']
submit_df = pd.concat([submit_df, tmp_df], axis=0)
 submit_df
%%time
y_train_full = df_train['isFraud'].copy()
X_train_full = df_train.drop('isFraud', axis=1)
 X_test = df_test.copy()<split>
category_df = train_test_df[train_test_df.split_category_0 == 'Men']
category_df.head()
%%time
X_train, X_val, y_train, Y_val = train_test_split(X_train_full, y_train_full,
test_size=0.15,
random_state=42 )<train_model>
name = category_df.name.values
item_desc = category_df.item_description.values
vectorizer = TfidfVectorizer(max_df=0.85, min_df=2, ngram_range=(1,2), max_features=100000)
 name_vecs = vectorizer.fit_transform(name)
 item_desc_vecs = vectorizer.fit_transform(item_desc )
%%time
clf = RandomForestClassifier(n_estimators=1000, max_depth=2, random_state=0, n_jobs=4)
 clf.fit(X_train, y_train)
<save_to_csv>
category_df = category_df.drop(['name', 'item_description', 'split_category_0'], axis=1)
 category_df.split_category_1 = category_df.split_category_1.astype('category')
 category_df.split_category_2 = category_df.split_category_2.astype('category')
 category_df.brand_name = category_df.brand_name.astype('category')
 category_df.brand_category_2 = category_df.brand_category_2.astype('category')
 category_df.split_category_1 = category_df.split_category_1.cat.codes
category_df.split_category_2 = category_df.split_category_2.cat.codes
category_df.brand_name = category_df.brand_name.cat.codes
category_df.brand_category_2 = category_df.brand_category_2.cat.codes
%%time
sample_submission['isFraud'] = clf.predict_proba(X_test)[:,1]
sample_submission.to_csv('RandomForest.csv' )<drop_column>
encoder = OneHotEncoder()
X_data = encoder.fit_transform(category_df)
 X_data = hstack([X_data, name_vecs, item_desc_vecs] ).tocsr()
X_data = np.log1p(X_data)
 y_train = np.log1p(train_df[train_df.split_category_0 == 'Men'].price)
 category_index = category_df.index
train_category = train_df.split_category_0.value_counts() ['Men']
submit_index = category_index[train_category:]
X_train = X_data[:train_category]
X_submit = X_data[train_category:]
del X_train, X_val, y_train, Y_val
gc.collect()<feature_engineering>
ridge = Ridge(alpha=3.1)
 ridge.fit(X_train, y_train )
%%time
df_train['Trans_min_mean'] = df_train['TransactionAmt'] - df_train['TransactionAmt'].mean()
df_train['Trans_min_std'] = df_train['Trans_min_mean'] / df_train['TransactionAmt'].std()
df_test['Trans_min_mean'] = df_test['TransactionAmt'] - df_test['TransactionAmt'].mean()
df_test['Trans_min_std'] = df_test['Trans_min_mean'] / df_test['TransactionAmt'].std()
df_train['TransactionAmt_to_mean_card1'] = df_train['TransactionAmt'] / df_train.groupby(['card1'])['TransactionAmt'].transform('mean')
 df_train['TransactionAmt_to_mean_card4'] = df_train['TransactionAmt'] / df_train.groupby(['card4'])['TransactionAmt'].transform('mean')
 df_train['TransactionAmt_to_std_card1'] = df_train['TransactionAmt'] / df_train.groupby(['card1'])['TransactionAmt'].transform('std')
 df_train['TransactionAmt_to_std_card4'] = df_train['TransactionAmt'] / df_train.groupby(['card4'])['TransactionAmt'].transform('std')
 df_test['TransactionAmt_to_mean_card1'] = df_test['TransactionAmt'] / df_test.groupby(['card1'])['TransactionAmt'].transform('mean')
 df_test['TransactionAmt_to_mean_card4'] = df_test['TransactionAmt'] / df_test.groupby(['card4'])['TransactionAmt'].transform('mean')
 df_test['TransactionAmt_to_std_card1'] = df_test['TransactionAmt'] / df_test.groupby(['card1'])['TransactionAmt'].transform('std')
 df_test['TransactionAmt_to_std_card4'] = df_test['TransactionAmt'] / df_test.groupby(['card4'])['TransactionAmt'].transform('std')
 df_train['TransactionAmt'] = np.log(df_train['TransactionAmt'])
 df_test['TransactionAmt'] = np.log(df_test['TransactionAmt'] )<concatenate>
price = np.expm1(ridge.predict(X_submit))
%%time
df_test['isFraud'] = 'test'
df = pd.concat([df_train, df_test], axis=0, sort=False)
 df = df.reset_index()
df = df.drop('index', axis=1 )<drop_column>
tmp_df = pd.DataFrame(submit_index)
 tmp_df = pd.concat([tmp_df, pd.DataFrame(price)], axis=1)
 tmp_df.columns = ['test_id', 'price']
submit_df = pd.concat([submit_df, tmp_df], axis=0)
 submit_df
def PCA_change(df, cols, n_components, prefix='PCA_', rand_seed=4):
pca = PCA(n_components=n_components, random_state=rand_seed)
 principalComponents = pca.fit_transform(df[cols])
 principalDf = pd.DataFrame(principalComponents)
 df.drop(cols, axis=1, inplace=True)
 principalDf.rename(columns=lambda x: str(prefix)+str(x), inplace=True)
 df = pd.concat([df, principalDf], axis=1)
 return df<normalization>
category_df = train_test_df[train_test_df.split_category_0 == 'Home']
category_df.head()
%%time
for col in mas_v:
df[col] = df[col].fillna(( df[col].min() - 2))
df[col] =(minmax_scale(df[col], feature_range=(0,1)))
 df = PCA_change(df, mas_v, prefix='PCA_V_', n_components=30)
<drop_column>
name = category_df.name.values
item_desc = category_df.item_description.values
vectorizer = TfidfVectorizer(max_df=0.85, min_df=2, ngram_range=(1,2), max_features=100000)
 name_vecs = vectorizer.fit_transform(name)
 item_desc_vecs = vectorizer.fit_transform(item_desc )
%%time
df_train, df_test = df[df['isFraud'] != 'test'], df[df['isFraud'] == 'test'].drop('isFraud', axis=1 )<drop_column>
category_df = category_df.drop(['name', 'item_description', 'split_category_0'], axis=1)
 category_df.split_category_1 = category_df.split_category_1.astype('category')
 category_df.split_category_2 = category_df.split_category_2.astype('category')
 category_df.brand_name = category_df.brand_name.astype('category')
 category_df.brand_category_2 = category_df.brand_category_2.astype('category')
 category_df.split_category_1 = category_df.split_category_1.cat.codes
category_df.split_category_2 = category_df.split_category_2.cat.codes
category_df.brand_name = category_df.brand_name.cat.codes
category_df.brand_category_2 = category_df.brand_category_2.cat.codes
category_df.head()
%%time
X_train = df_train.sort_values('TransactionDT' ).drop(['isFraud',
'TransactionDT',
],
axis=1)
 y_train = df_train.sort_values('TransactionDT')['isFraud'].astype(bool)
 X_test = df_test.sort_values('TransactionDT' ).drop(['TransactionDT',
],
axis=1)
 del df_train
df_test = df_test[["TransactionDT"]]<train_model>
encoder = OneHotEncoder()
X_data = encoder.fit_transform(category_df)
 X_data = hstack([X_data, name_vecs, item_desc_vecs] ).tocsr()
X_data = np.log1p(X_data)
 y_train = np.log1p(train_df[train_df.split_category_0 == 'Home'].price)
 category_index = category_df.index
train_category = train_df.split_category_0.value_counts() ['Home']
submit_index = category_index[train_category:]
X_train = X_data[:train_category]
X_submit = X_data[train_category:]
def objective(params):
time1 = time.time()
params = {
'max_depth': int(params['max_depth']),
'gamma': "{:.3f}".format(params['gamma']),
'subsample': "{:.2f}".format(params['subsample']),
'reg_alpha': "{:.3f}".format(params['reg_alpha']),
'reg_lambda': "{:.3f}".format(params['reg_lambda']),
'learning_rate': "{:.3f}".format(params['learning_rate']),
'num_leaves': '{:.3f}'.format(params['num_leaves']),
'colsample_bytree': '{:.3f}'.format(params['colsample_bytree']),
'min_child_samples': '{:.3f}'.format(params['min_child_samples']),
'feature_fraction': '{:.3f}'.format(params['feature_fraction']),
'bagging_fraction': '{:.3f}'.format(params['bagging_fraction'])
 }
print("
print(f"params = {params}")
 FOLDS = 7
count=1
skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=42)
 tss = TimeSeriesSplit(n_splits=FOLDS)
 y_preds = np.zeros(sample_submission.shape[0])
 y_oof = np.zeros(X_train.shape[0])
 score_mean = 0
for tr_idx, val_idx in tss.split(X_train, y_train):
clf = xgb.XGBClassifier(
 n_estimators=600, random_state=4, verbose=True,
tree_method='gpu_hist',
**params 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 score = make_scorer(roc_auc_score, needs_proba=True )(clf, X_vl, y_vl)
 score_mean += score
print(f'{count} CV - score: {round(score, 4)}')
 count += 1
time2 = time.time() - time1
print(f"Total Time Run: {round(time2 / 60,2)}")
 print(f'Mean ROC_AUC: {score_mean / FOLDS}')
 del X_tr, X_vl, y_tr, y_vl, clf, score
gc.collect()
return -(score_mean / FOLDS)
 space = {
'max_depth': hp.quniform('max_depth', 7, 23, 1),
'reg_alpha': hp.uniform('reg_alpha', 0.01, 0.4),
'reg_lambda': hp.uniform('reg_lambda', 0.01,.4),
'learning_rate': hp.uniform('learning_rate', 0.01, 0.2),
'colsample_bytree': hp.uniform('colsample_bytree', 0.3,.9),
'gamma': hp.uniform('gamma', 0.01,.7),
'num_leaves': hp.choice('num_leaves', list(range(20, 250, 10))),
'min_child_samples': hp.choice('min_child_samples', list(range(100, 250, 10))),
'subsample': hp.choice('subsample', [0.2, 0.4, 0.5, 0.6, 0.7,.8,.9]),
'feature_fraction': hp.uniform('feature_fraction', 0.4,.8),
'bagging_fraction': hp.uniform('bagging_fraction', 0.4,.9)
 } 
<set_options>
ridge = Ridge(alpha=3.1)
 ridge.fit(X_train, y_train )
gc.collect()<find_best_params>
price = np.expm1(ridge.predict(X_submit))
%%time
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=5)
 best_params = space_eval(space, best )<train_model>
tmp_df = pd.DataFrame(submit_index)
 tmp_df = pd.concat([tmp_df, pd.DataFrame(price)], axis=1)
 tmp_df.columns = ['test_id', 'price']
submit_df = pd.concat([submit_df, tmp_df], axis=0)
 submit_df
%%time
print("BEST PARAMS: ", best_params)
 best_params['max_depth'] = int(best_params['max_depth'] )<choose_model_class>
category_df = train_test_df[train_test_df.split_category_0 == 'Vintage & Collectibles']
category_df.head()
%%time
clf = xgb.XGBClassifier(
 n_estimators=300,
**best_params,
tree_method='gpu_hist'
)<compute_train_metric>
name = category_df.name.values
item_desc = category_df.item_description.values
vectorizer = TfidfVectorizer(max_df=0.85, min_df=2, ngram_range=(1,2), max_features=100000)
 name_vecs = vectorizer.fit_transform(name)
 item_desc_vecs = vectorizer.fit_transform(item_desc )
%%time
kfold = KFold(n_splits=10, random_state=7)
 results = cross_val_score(clf, X_train, y_train, cv=kfold)
 print("Accuracy: %.2f%%(%.2f%%)" %(results.mean() *100, results.std() *100))<predict_on_test>
category_df = category_df.drop(['name', 'item_description', 'split_category_0'], axis=1)
 category_df.split_category_1 = category_df.split_category_1.astype('category')
 category_df.split_category_2 = category_df.split_category_2.astype('category')
 category_df.brand_name = category_df.brand_name.astype('category')
 category_df.brand_category_2 = category_df.brand_category_2.astype('category')
 category_df.split_category_1 = category_df.split_category_1.cat.codes
category_df.split_category_2 = category_df.split_category_2.cat.codes
category_df.brand_name = category_df.brand_name.cat.codes
category_df.brand_category_2 = category_df.brand_category_2.cat.codes
category_df.head()
%%time
clf.fit(X_train, y_train)
 y_preds = clf.predict_proba(X_test)[:,1]<save_to_csv>
encoder = OneHotEncoder()
X_data = encoder.fit_transform(category_df)
 X_data = hstack([X_data, name_vecs, item_desc_vecs] ).tocsr()
X_data = np.log1p(X_data)
 y_train = np.log1p(train_df[train_df.split_category_0 == 'Vintage & Collectibles'].price)
 category_index = category_df.index
train_category = train_df.split_category_0.value_counts() ['Vintage & Collectibles']
submit_index = category_index[train_category:]
X_train = X_data[:train_category]
X_submit = X_data[train_category:]
sample_submission['isFraud'] = y_preds
sample_submission.to_csv('XGB_hypopt_model.csv' )<define_variables>
ridge = Ridge(alpha=3.1)
 ridge.fit(X_train, y_train )
<init_hyperparams>
price = np.expm1(ridge.predict(X_submit))
<load_from_csv>
tmp_df = pd.DataFrame(submit_index)
 tmp_df = pd.concat([tmp_df, pd.DataFrame(price)], axis=1)
 tmp_df.columns = ['test_id', 'price']
submit_df = pd.concat([submit_df, tmp_df], axis=0)
 submit_df
id_name = 'TransactionID'
label_name = 'isFraud'
submission = pd.read_csv('/kaggle/input/ieee-fraud-detection/sample_submission.csv')
 def Merge_sub(weightDic):
plt.figure(figsize=(14,7))
all_sub = np.zeros(submission.shape[0])
 for suffix in weightDic.keys() :
sub = pd.read_csv('/kaggle/input/10th-submissionpart-daishu/submission_%s.csv'%(suffix))
sns.distplot(np.log10(sub[label_name]),bins=200,label=suffix)
 all_sub += sub[label_name] * weightDic[suffix]
sub[label_name] = all_sub
sns.distplot(np.log10(sub[label_name]),bins=200,label='after merge')
 plt.legend()
sub.to_csv('./submission.csv',index=False)
 return None<set_options>
category_df = train_test_df[train_test_df.split_category_0 == 'Other']
category_df.head()
warnings.simplefilter('ignore')
 sns.set()
%matplotlib inline
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 SEED = 1993
seed_everything(SEED)
 START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d' )<define_variables>
name = category_df.name.values
item_desc = category_df.item_description.values
vectorizer = TfidfVectorizer(max_df=0.85, min_df=2, ngram_range=(1,2), max_features=100000)
 name_vecs = vectorizer.fit_transform(name)
 item_desc_vecs = vectorizer.fit_transform(item_desc )
files = ['.. /input/ieee-fraud-detection/test_identity.csv',
'.. /input/ieee-fraud-detection/test_transaction.csv',
'.. /input/ieee-fraud-detection/train_identity.csv',
'.. /input/ieee-fraud-detection/train_transaction.csv',
'.. /input/ieee-fraud-detection/sample_submission.csv']<load_from_csv>
category_df = category_df.drop(['name', 'item_description', 'split_category_0'], axis=1)
 category_df.split_category_1 = category_df.split_category_1.astype('category')
 category_df.split_category_2 = category_df.split_category_2.astype('category')
 category_df.brand_name = category_df.brand_name.astype('category')
 category_df.brand_category_2 = category_df.brand_category_2.astype('category')
 category_df.split_category_1 = category_df.split_category_1.cat.codes
category_df.split_category_2 = category_df.split_category_2.cat.codes
category_df.brand_name = category_df.brand_name.cat.codes
category_df.brand_category_2 = category_df.brand_category_2.cat.codes
category_df.head()
%%time
def load_data(file):
return pd.read_csv(file)
 with multiprocessing.Pool() as pool:
test_identity, test_transaction, train_identity, train_transaction, sample_submission = pool.map(load_data, files )<merge>
encoder = OneHotEncoder()
X_data = encoder.fit_transform(category_df)
 X_data = hstack([X_data, name_vecs, item_desc_vecs] ).tocsr()
X_data = np.log1p(X_data)
 y_train = np.log1p(train_df[train_df.split_category_0 == 'Other'].price)
 category_index = category_df.index
train_category = train_df.split_category_0.value_counts() ['Other']
submit_index = category_index[train_category:]
X_train = X_data[:train_category]
X_submit = X_data[train_category:]
train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left')
 sample_submission = sample_submission.set_index('TransactionID')
 print("Train shape: ", train.shape)
 print("Test shape:", test.shape)
 y = train['isFraud'].copy()
del train_transaction, train_identity, test_transaction, test_identity
gc.collect()
train = train.drop('isFraud', axis=1 )<drop_column>
ridge = Ridge(alpha=3.1)
 ridge.fit(X_train, y_train )
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<define_variables>
price = np.expm1(ridge.predict(X_submit))
useful_features = ['TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13',
'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M2', 'M3',
'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V17',
'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48',
'V49', 'V51', 'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V69', 'V70', 'V71',
'V72', 'V73', 'V74', 'V75', 'V76', 'V78', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128', 'V130', 'V131', 'V138', 'V139', 'V140',
'V143', 'V145', 'V146', 'V147', 'V149', 'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161',
'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173', 'V175', 'V176', 'V177',
'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189', 'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204',
'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219', 'V220',
'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V231', 'V233', 'V234', 'V238', 'V239',
'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261',
'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276',
'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285', 'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303',
'V304', 'V306', 'V307', 'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324', 'V326',
'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338', 'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09',
'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33',
'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo']<drop_column>
tmp_df = pd.DataFrame(submit_index)
 tmp_df = pd.concat([tmp_df, pd.DataFrame(price)], axis=1)
 tmp_df.columns = ['test_id', 'price']
submit_df = pd.concat([submit_df, tmp_df], axis=0)
 submit_df.shape
cols_to_drop = [col for col in train.columns if col not in useful_features]
cols_to_drop.remove('TransactionID')
 cols_to_drop.remove('TransactionDT' )<drop_column>
name = category_df.name.values
item_desc = category_df.item_description.values
vectorizer = TfidfVectorizer(max_df=0.85, min_df=2, ngram_range=(1,2), max_features=100000)
 name_vecs = vectorizer.fit_transform(name)
 item_desc_vecs = vectorizer.fit_transform(item_desc )
print('{} features are going to be dropped for being useless'.format(len(cols_to_drop)))
 train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )<data_type_conversions>
category_df = category_df.drop(['name', 'item_description', 'split_category_0'], axis=1)
 category_df.split_category_1 = category_df.split_category_1.astype('category')
 category_df.split_category_2 = category_df.split_category_2.astype('category')
 category_df.brand_name = category_df.brand_name.astype('category')
 category_df.brand_category_2 = category_df.brand_category_2.astype('category')
 category_df.split_category_1 = category_df.split_category_1.cat.codes
category_df.split_category_2 = category_df.split_category_2.cat.codes
category_df.brand_name = category_df.brand_name.cat.codes
category_df.brand_category_2 = category_df.brand_category_2.cat.codes
category_df.head()
def addNewFeatures(data):
data['uid1'] = data['card1'].astype(str)+'_'+data['card2'].astype(str)
 data['uid2'] = data['uid1'].astype(str)+'_'+data['card3'].astype(str)+'_'+data['card5'].astype(str)
 data['uid3'] = data['uid2'].astype(str)+'_'+data['addr1'].astype(str)+'_'+data['addr2'].astype(str)
 data['D9'] = np.where(data['D9'].isna() ,0,1)
 return data
train = addNewFeatures(train)
 test = addNewFeatures(test)
 train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour_of_day'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour_of_day'] = np.floor(test['TransactionDT'] / 3600)% 24
train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24
train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
for feature in ['id_34', 'id_36']:
if feature in useful_features:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_35', 'id_36']:
if feature in useful_features:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))<count_values>
encoder = OneHotEncoder()
X_data = encoder.fit_transform(category_df)
 X_data = hstack([X_data, name_vecs, item_desc_vecs] ).tocsr()
X_data = np.log1p(X_data)
 y_train = np.log1p(train_df[train_df.split_category_0 == 'Handmade'].price)
 category_index = category_df.index
train_category = train_df.split_category_0.value_counts() ['Handmade']
submit_index = category_index[train_category:]
X_train = X_data[:train_category]
X_submit = X_data[train_category:]
train['card1_count_full'] = train['card1'].map(pd.concat([train['card1'], test['card1']], ignore_index=True ).value_counts(dropna=False))
test['card1_count_full'] = test['card1'].map(pd.concat([train['card1'], test['card1']], ignore_index=True ).value_counts(dropna=False))
train['card2_count_full'] = train['card2'].map(pd.concat([train['card2'], test['card2']], ignore_index=True ).value_counts(dropna=False))
test['card2_count_full'] = test['card2'].map(pd.concat([train['card2'], test['card2']], ignore_index=True ).value_counts(dropna=False))
train['card3_count_full'] = train['card3'].map(pd.concat([train['card3'], test['card3']], ignore_index=True ).value_counts(dropna=False))
test['card3_count_full'] = test['card3'].map(pd.concat([train['card3'], test['card3']], ignore_index=True ).value_counts(dropna=False))
train['card4_count_full'] = train['card4'].map(pd.concat([train['card4'], test['card4']], ignore_index=True ).value_counts(dropna=False))
test['card4_count_full'] = test['card4'].map(pd.concat([train['card4'], test['card4']], ignore_index=True ).value_counts(dropna=False))
train['card5_count_full'] = train['card5'].map(pd.concat([train['card5'], test['card5']], ignore_index=True ).value_counts(dropna=False))
test['card5_count_full'] = test['card5'].map(pd.concat([train['card5'], test['card5']], ignore_index=True ).value_counts(dropna=False))
train['card6_count_full'] = train['card6'].map(pd.concat([train['card6'], test['card6']], ignore_index=True ).value_counts(dropna=False))
test['card6_count_full'] = test['card6'].map(pd.concat([train['card6'], test['card6']], ignore_index=True ).value_counts(dropna=False))
train['addr1_count_full'] = train['addr1'].map(pd.concat([train['addr1'], test['addr1']], ignore_index=True ).value_counts(dropna=False))
test['addr1_count_full'] = test['addr1'].map(pd.concat([train['addr1'], test['addr1']], ignore_index=True ).value_counts(dropna=False))
train['addr2_count_full'] = train['addr2'].map(pd.concat([train['addr2'], test['addr2']], ignore_index=True ).value_counts(dropna=False))
test['addr2_count_full'] = test['addr2'].map(pd.concat([train['addr2'], test['addr2']], ignore_index=True ).value_counts(dropna=False))<categorify>
ridge = Ridge(alpha=3.1)
 ridge.fit(X_train, y_train )
train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')
 train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')
 train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')
 train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')
 train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')
 train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')
 test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')
 test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')
 test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')
 test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std' )<feature_engineering>
price = np.expm1(ridge.predict(X_submit))
train['TransactionAmt'] = np.log1p(train['TransactionAmt'])
 test['TransactionAmt'] = np.log1p(test['TransactionAmt'] )<data_type_conversions>
tmp_df = pd.DataFrame(submit_index)
 tmp_df = pd.concat([tmp_df, pd.DataFrame(price)], axis=1)
 tmp_df.columns = ['test_id', 'price']
submit_df = pd.concat([submit_df, tmp_df], axis=0)
 submit_df.shape
dates_range = pd.date_range(start='2017-10-01', end='2019-01-01')
 us_holidays = calendar().holidays(start=dates_range.min() , end=dates_range.max())
 def setTime(df):
df['DT'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 df['DT_M'] =(( df['DT'].dt.year-2017)*12 + df['DT'].dt.month ).astype(np.int8)
 df['DT_W'] =(( df['DT'].dt.year-2017)*52 + df['DT'].dt.weekofyear ).astype(np.int8)
 df['DT_D'] =(( df['DT'].dt.year-2017)*365 + df['DT'].dt.dayofyear ).astype(np.int16)
 df['DT_hour'] =(df['DT'].dt.hour ).astype(np.int8)
 df['DT_day_week'] =(df['DT'].dt.dayofweek ).astype(np.int8)
 df['DT_day_month'] =(df['DT'].dt.day ).astype(np.int8)
 df['is_december'] = df['DT'].dt.month
df['is_december'] =(df['is_december']==12 ).astype(np.int8)
 df['is_holiday'] =(df['DT'].dt.date.astype('datetime64' ).isin(us_holidays)).astype(np.int8)
 return df
train = setTime(train)
 test = setTime(test )<data_type_conversions>
category_df = train_test_df[train_test_df.split_category_0 == 'Sports & Outdoors']
category_df.head()
i_cols = ['card1','card2','card3','card5','uid1','uid2','uid3']
for col in i_cols:
for agg_type in ['mean','std']:
new_col_name = col+'_TransactionAmt_'+agg_type
temp_df = pd.concat([train[[col, 'TransactionAmt']], test[[col,'TransactionAmt']]])
 temp_df = temp_df.groupby([col])['TransactionAmt'].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train[new_col_name] = train[col].map(temp_df)
 test[new_col_name] = test[col].map(temp_df)
 train = train.replace(np.inf,999)
 test = test.replace(np.inf,999 )<feature_engineering>
name = category_df.name.values
item_desc = category_df.item_description.values
vectorizer = TfidfVectorizer(max_df=0.85, min_df=2, ngram_range=(1,2), max_features=100000)
 name_vecs = vectorizer.fit_transform(name)
 item_desc_vecs = vectorizer.fit_transform(item_desc )
train['P_isproton'] =(train['P_emaildomain']=='protonmail.com')
 train['R_isproton'] =(train['R_emaildomain']=='protonmail.com')
 test['P_isproton'] =(test['P_emaildomain']=='protonmail.com')
 test['R_isproton'] =(test['R_emaildomain']=='protonmail.com' )<count_missing_values>
category_df = category_df.drop(['name', 'item_description', 'split_category_0'], axis=1)
 category_df.split_category_1 = category_df.split_category_1.astype('category')
 category_df.split_category_2 = category_df.split_category_2.astype('category')
 category_df.brand_name = category_df.brand_name.astype('category')
 category_df.brand_category_2 = category_df.brand_category_2.astype('category')
 category_df.split_category_1 = category_df.split_category_1.cat.codes
category_df.split_category_2 = category_df.split_category_2.cat.codes
category_df.brand_name = category_df.brand_name.cat.codes
category_df.brand_category_2 = category_df.brand_category_2.cat.codes
category_df.head()
train['nulls1'] = train.isna().sum(axis=1)
 test['nulls1'] = test.isna().sum(axis=1 )<define_variables>
encoder = OneHotEncoder()
X_data = encoder.fit_transform(category_df)
 X_data = hstack([X_data, name_vecs, item_desc_vecs] ).tocsr()
X_data = np.log1p(X_data)
 y_train = np.log1p(train_df[train_df.split_category_0 == 'Sports & Outdoors'].price)
 category_index = category_df.index
train_category = train_df.split_category_0.value_counts() ['Sports & Outdoors']
submit_index = category_index[train_category:]
X_train = X_data[:train_category]
X_submit = X_data[train_category:]
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum', 'scranton.edu': 'other', 'optonline.net': 'other',
'hotmail.co.uk': 'microsoft', 'comcast.net': 'other', 'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo', 'charter.net': 'spectrum', 'live.com': 'microsoft', 'aim.com': 'aol', 'hotmail.de': 'microsoft',
'centurylink.net': 'centurylink', 'gmail.com': 'google', 'me.com': 'apple', 'earthlink.net': 'other',
'gmx.de': 'other', 'web.de': 'other', 'cfl.rr.com': 'other', 'hotmail.com': 'microsoft', 'protonmail.com': 'other',
'hotmail.fr': 'microsoft', 'windstream.net': 'other', 'outlook.es': 'microsoft', 'yahoo.co.jp': 'yahoo',
'yahoo.de': 'yahoo', 'servicios-ta.com': 'other', 'netzero.net': 'other', 'suddenlink.net': 'other',
'roadrunner.com': 'other', 'sc.rr.com': 'other', 'live.fr': 'microsoft', 'verizon.net': 'yahoo',
'msn.com': 'microsoft', 'q.com': 'centurylink', 'prodigy.net.mx': 'att', 'frontier.com': 'yahoo',
'anonymous.com': 'other', 'rocketmail.com': 'yahoo', 'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo', 'outlook.com': 'microsoft', 'mail.com': 'other', 'bellsouth.net': 'other',
'embarqmail.com': 'centurylink', 'cableone.net': 'other', 'hotmail.es': 'microsoft', 'mac.com': 'apple',
'yahoo.co.uk': 'yahoo', 'netzero.com': 'other', 'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other',
'cox.net': 'other', 'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']<feature_engineering>
ridge = Ridge(alpha=3.1)
 ridge.fit(X_train, y_train )
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )<feature_engineering>
price = np.expm1(ridge.predict(X_submit))
p = 'P_emaildomain'
r = 'R_emaildomain'
uknown = 'email_not_provided'
def setDomain(df):
df[p] = df[p].fillna(uknown)
 df[r] = df[r].fillna(uknown)
 df['email_check'] = np.where(( df[p]==df[r])&(df[p]!=uknown),1,0)
 df[p+'_prefix'] = df[p].apply(lambda x: str(x ).split('.')[0])
 df[r+'_prefix'] = df[r].apply(lambda x: str(x ).split('.')[0])
 return df
train=setDomain(train)
 test=setDomain(test )<feature_engineering>
tmp_df = pd.DataFrame(submit_index)
 tmp_df = pd.concat([tmp_df, pd.DataFrame(price)], axis=1)
 tmp_df.columns = ['test_id', 'price']
submit_df = pd.concat([submit_df, tmp_df], axis=0)
 submit_df.shape
train["lastest_browser"] = np.zeros(train.shape[0])
 test["lastest_browser"] = np.zeros(test.shape[0])
 def setBrowser(df):
df.loc[df["id_31"] == "samsung browser 7.0",'lastest_browser']=1
df.loc[df["id_31"] == "opera 53.0",'lastest_browser']=1
df.loc[df["id_31"] == "mobile safari 10.0",'lastest_browser']=1
df.loc[df["id_31"] == "google search application 49.0",'lastest_browser']=1
df.loc[df["id_31"] == "firefox 60.0",'lastest_browser']=1
df.loc[df["id_31"] == "edge 17.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 69.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 67.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 63.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 63.0 for ios",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 64.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 64.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 64.0 for ios",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 65.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 65.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 65.0 for ios",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 66.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 66.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 66.0 for ios",'lastest_browser']=1
return df
train = setBrowser(train)
 test = setBrowser(test )<feature_engineering>
name = category_df.name.values
item_desc = category_df.item_description.values
vectorizer = TfidfVectorizer(max_df=0.85, min_df=2, ngram_range=(1,2), max_features=100000)
 name_vecs = vectorizer.fit_transform(name)
 item_desc_vecs = vectorizer.fit_transform(item_desc )
def setDevice(df):
df['DeviceInfo'] = df['DeviceInfo'].fillna('unknown_device' ).str.lower()
df['device_name'] = df['DeviceInfo'].str.split('/', expand=True)[0]
df.loc[df['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
df.loc[df['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
df.loc[df['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
df.loc[df['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
df.loc[df['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
df.loc[df['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
df.loc[df['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
df.loc[df['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
df.loc[df.device_name.isin(df.device_name.value_counts() [df.device_name.value_counts() < 200].index), 'device_name'] = "Others"
df['had_id'] = 1
gc.collect()
return df
train = setDevice(train)
 test = setDevice(test )<concatenate>
category_df = category_df.drop(['name', 'item_description', 'split_category_0'], axis=1)
 category_df.split_category_1 = category_df.split_category_1.astype('category')
 category_df.split_category_2 = category_df.split_category_2.astype('category')
 category_df.brand_name = category_df.brand_name.astype('category')
 category_df.brand_category_2 = category_df.brand_category_2.astype('category')
 category_df.split_category_1 = category_df.split_category_1.cat.codes
category_df.split_category_2 = category_df.split_category_2.cat.codes
category_df.brand_name = category_df.brand_name.cat.codes
category_df.brand_category_2 = category_df.brand_category_2.cat.codes
category_df.head()
i_cols = ['card1','card2','card3','card5',
'C1','C2','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D8',
'addr1','addr2',
'dist1',
'P_emaildomain', 'R_emaildomain',
'DeviceInfo','device_name',
'id_30','id_33',
'uid1','uid2','uid3',
]
for col in i_cols:
temp_df = pd.concat([train[[col]], test[[col]]])
 fq_encode = temp_df[col].value_counts(dropna=False ).to_dict()
train[col+'_fq_enc'] = train[col].map(fq_encode)
 test[col+'_fq_enc'] = test[col].map(fq_encode)
 for col in ['DT_M','DT_W','DT_D']:
temp_df = pd.concat([train[[col]], test[[col]]])
 fq_encode = temp_df[col].value_counts().to_dict()
train[col+'_total'] = train[col].map(fq_encode)
 test[col+'_total'] = test[col].map(fq_encode)
 periods = ['DT_M','DT_W','DT_D']
i_cols = ['uid1']
for period in periods:
for col in i_cols:
new_column = col + '_' + period
temp_df = pd.concat([train[[col,period]], test[[col,period]]])
 temp_df[new_column] = temp_df[col].astype(str)+ '_' +(temp_df[period] ).astype(str)
 fq_encode = temp_df[new_column].value_counts().to_dict()
train[new_column] =(train[col].astype(str)+ '_' + train[period].astype(str)).map(fq_encode)
 test[new_column] =(test[col].astype(str)+ '_' + test[period].astype(str)).map(fq_encode)
 train[new_column] /= train[period+'_total']
test[new_column] /= test[period+'_total']<statistical_test>
encoder = OneHotEncoder()
X_data = encoder.fit_transform(category_df)
 X_data = hstack([X_data, name_vecs, item_desc_vecs] ).tocsr()
X_data = np.log1p(X_data)
 y_train = np.log1p(train_df[train_df.split_category_0 == 'Nan'].price)
 category_index = category_df.index
train_category = train_df.split_category_0.value_counts() ['Nan']
submit_index = category_index[train_category:]
X_train = X_data[:train_category]
X_submit = X_data[train_category:]
def get_too_many_null_attr(data):
many_null_cols = [col for col in data.columns if data[col].isnull().sum() / data.shape[0] > 0.9]
return many_null_cols
def get_too_many_repeated_val(data):
big_top_value_cols = [col for col in train.columns if train[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
return big_top_value_cols
def get_useless_columns(data):
too_many_null = get_too_many_null_attr(data)
 print("More than 90% null: " + str(len(too_many_null)))
 too_many_repeated = get_too_many_repeated_val(data)
 print("More than 90% repeated value: " + str(len(too_many_repeated)))
 cols_to_drop = list(set(too_many_null + too_many_repeated))
return cols_to_drop<categorify>
ridge = Ridge(alpha=3.1)
 ridge.fit(X_train, y_train )
for f in train.columns:
if train[f].dtype.name =='object' or test[f].dtype.name =='object':
lbl = LabelEncoder()
lbl.fit(list(train[f].values)+ list(test[f].values))
train[f] = lbl.transform(list(train[f].values))
test[f] = lbl.transform(list(test[f].values))<categorify>
price = np.expm1(ridge.predict(X_submit))
train = train.fillna(-999)
 test = test.fillna(-999 )<count_missing_values>
tmp_df = pd.DataFrame(submit_index)
 tmp_df = pd.concat([tmp_df, pd.DataFrame(price)], axis=1)
 tmp_df.columns = ['test_id', 'price']
submit_df = pd.concat([submit_df, tmp_df], axis=0)
 submit_df.shape
print(train.isnull().sum().max())
 print(test.isnull().sum().max() )<drop_column>
submit_df = submit_df.sort_values(by=['test_id'], ascending=True )
<train_on_grid><EOS> 
submit_df.to_csv('submission.csv', index=False )