Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<count_unique_values>
warnings.filterwarnings("ignore")
 %load_ext autoreload
%autoreload
def drop_messy_cols(df):
numeric_cols=list(df.select_dtypes(include=['int32','int64','float32'] ).columns)
 categorical_cols=list(df.select_dtypes(include=['object'] ).columns)
 bool_cols=list(df.select_dtypes(include=['bool'] ).columns)

low_cardinality_cols=[col for col in categorical_cols if df[col].nunique() <10]
print('
',df[low_cardinality_cols].nunique())

high_cardinality_cols=[col for col in categorical_cols if df[col].nunique() >10 and col not in [
'totals.transactionRevenue','fullVisitorId', 'totals.hits', 'device.operatingSystem', 'geoNetwork.metro'
'totals.pageviews','trafficSource.source']
]
print('
',df[high_cardinality_cols].nunique())

high_na_cols=[col for col in df.columns if df[col].isna().sum() >10000 and col not in ['totals.transactionRevenue']]
print('
',df[high_na_cols].isna().sum())
 _COLS_TO_DROP = ['visitNumber', 'totals.newVisits', 'totals.bounces', 'trafficSource.isTrueDirect', 'geoNetwork.city', 'geoNetwork.networkDomain',
'geoNetwork.region', 'geoNetwork.subContinent', 'trafficSource.adContent', 'trafficSource.campaign', 'trafficSource.keyword',
'trafficSource.referralPath', 'device.isMobile'
]
df.drop(columns=_COLS_TO_DROP , inplace=True, axis=1)
 return df
def preprocess_and_impute(train_df, test_df):
full_df=pd.concat([train_df,test_df],axis=0)

full_df=drop_messy_cols(full_df)

_CNTRYS_LT_4_TRANSACTIONS = list(full_df.groupby('geoNetwork.country')['totals.transactions'].sum() [full_df.groupby('geoNetwork.country')['totals.transactions'].sum() <4].index)
 full_df['geoNetwork.country'][full_df['geoNetwork.country'].isin(_CNTRYS_LT_4_TRANSACTIONS)]='Other'

 _VALID_BROWSERS = [ 'Chrome', 'Safari', 'Firefox', 'Internet Explorer', 'Android Webview', 'Edge', 'Samsung Internet']
full_df['device.browser'][~full_df['device.browser'].isin(_VALID_BROWSERS)]='Other'

 _VALID_OPERATING_SYSTEMS = ['Windows', 'Macintosh', 'Android', 'iOS', 'Linux', 'Chrome OS', 'Windows Phone', 'Samsung']
full_df['device.operatingSystem'][~full_df['device.operatingSystem'].isin(_VALID_OPERATING_SYSTEMS)]='Other'

 _SPARSE_METROS= list(full_df.groupby('geoNetwork.metro')['totals.transactions'].sum() [full_df.groupby('geoNetwork.metro')['totals.transactions'].sum() <1].index)
 print('
Found {} sparse-metros.The entire dataset still has {} metros.'.format(len(set(_SPARSE_METROS)) , len(set(full_df['geoNetwork.metro'])) -len(set(_SPARSE_METROS))))
full_df['geoNetwork.metro'][full_df['geoNetwork.metro'].isin(_SPARSE_METROS)]='Other'

 _INEFFECTIVE_TRAFFIC_SOURCES=list(full_df.groupby('trafficSource.source')['totals.transactions'].sum() [full_df.groupby('trafficSource.source')['totals.transactions'].sum() <1].index)
 print('
Found {} ineffective traffic-sources.The entire dataset still has {} traffic-sources.'.format(len(set(_INEFFECTIVE_TRAFFIC_SOURCES)) , len(set(full_df['trafficSource.source'])) -len(set(_INEFFECTIVE_TRAFFIC_SOURCES))))
full_df['trafficSource.source'][full_df['trafficSource.source'].isin(_INEFFECTIVE_TRAFFIC_SOURCES)]='Other'
full_df['trafficSource.source'][full_df['trafficSource.source'].str.contains('google')]='google'
full_df['trafficSource.source'][full_df['trafficSource.source'].str.contains('facebook')]='facebook'
full_df['trafficSource.source'][full_df['trafficSource.source'].str.contains('yahoo')]='yahoo'
print('
LIST OF trafficSource.source: ',list(full_df['trafficSource.source'].value_counts().index))
full_df['date.year']=full_df['date'].dt.year
full_df['date.month']=full_df['date'].dt.month
full_df['date.weekday']=full_df['date'].dt.weekday
full_df['date.day']=full_df['date'].dt.day
del(full_df['date'])

_CAT_COLS_FOR_ENCODER = ['channelGrouping', 'date.year','date.month', 'device.browser', 'device.deviceCategory', 'device.operatingSystem',
'geoNetwork.continent', 'geoNetwork.country', 'geoNetwork.metro', 'trafficSource.medium', 'trafficSource.source']
for col in _CAT_COLS_FOR_ENCODER:
lb = LabelEncoder()
full_df[col]=lb.fit_transform(full_df[col])
 return full_df<load_from_csv>
model_path='.'
path='.. /input/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=64
num_workers=None
sz=96
def load_df_without_hits(path=dirname+'/train_v2.csv'):
ans=pd.DataFrame()
dfs=pd.read_csv(path, parse_dates=['date'],
converters={ j: json.loads for j in _JSON_COLUMNS },
dtype={'fullVisitorId':str}, chunksize=100000 
)
 for df in dfs:
df.reset_index(drop=True, inplace=True)

for col in _JSON_COLUMNS:
column_as_df= pd.json_normalize(df[col])
 column_as_df.columns=[f'{col}.{subcolumn}' for subcolumn in column_as_df.columns]
df=df.drop(col, axis=1 ).merge(column_as_df, right_index=True, left_index=True)

df=df.replace(_COL_VALUES_TO_BE_REPLACED , 'NA')

for col in _COLS_TO_CONVERT_TO_INT:
df[col] = df[col].fillna(1 ).astype(int)

df['totals.transactionRevenue'] = df['totals.transactionRevenue'].fillna(0 ).astype(float)
 df['totals.totalTransactionRevenue'] = df['totals.totalTransactionRevenue'].fillna(0 ).astype(float)

use_df = df[_FEATURES_WITHOUT_HITS]
del df
gc.collect()
ans = pd.concat([ans, use_df], axis = 0 ).reset_index(drop = True)
 if ans.shape[0]%200000==0:
print(f"Loaded {os.path.basename(path)} and now df= {ans.shape}")
 return ans

 _JSON_COLUMNS = [
'device','geoNetwork','totals','trafficSource'
]

 _FEATURES_WITHOUT_HITS = [
'channelGrouping', 'date', 'fullVisitorId', 'socialEngagementType', 'visitId', 'visitNumber', 'visitStartTime',
'device.browser', 'device.deviceCategory', 'device.isMobile', 'device.operatingSystem',
'geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country', 'geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region', 'geoNetwork.subContinent',
'totals.hits', 'totals.pageviews', 'totals.transactionRevenue','totals.totalTransactionRevenue', 'totals.sessionQualityDim', 'totals.transactions', 'totals.bounces', 'totals.newVisits', 'totals.timeOnSite',
'trafficSource.medium', 'trafficSource.source', 'trafficSource.isTrueDirect', 'trafficSource.keyword','trafficSource.campaign','trafficSource.referralPath', 'trafficSource.adContent',
'trafficSource.adwordsClickInfo.slot'
]

 _COLS_TO_CONVERT_TO_INT = [
'totals.pageviews', 'totals.hits', 'totals.sessionQualityDim',
'totals.visits', 'totals.timeOnSite', 'totals.transactions'
]

 _COL_VALUES_TO_BE_REPLACED = {
'device.operatingSystem' : ['not available in demo dataset','(not set)'],
'device.browser' : ['not available in demo dataset','(not set)','0'],
'geoNetwork.city' : ['not available in demo dataset','(not set)'],
'geoNetwork.metro' : ['not available in demo dataset', '(not set)'],
'geoNetwork.networkDomain' : ['unknown.unknown', '(not set)', 'not.configured'],
'geoNetwork.region' : ['not available in demo dataset','(not set)'],
'geoNetwork.continent' : ['not available in demo dataset','(not set)'],
'geoNetwork.subContinent' : ['not available in demo dataset','(not set)'],
'geoNetwork.country' : ['not available in demo dataset','(not set)'],
'trafficSource.campaign' : ['not available in demo dataset','(not set)'],
'trafficSource.medium' : ['(none)','(not set)'],
'trafficSource.source' : '(not set)'
}<load_from_csv>
df_trn=pd.read_csv(train_lbl )
%%time
print('{} Reading the 27 Gig 'train_v2.csv' will take ~18 minutes time! Grab a coffee in the meanwhile.{}
'.format('
train = load_df_without_hits(path=dirname+'/train_v2.csv')
 print(list(train.columns))
train_bkp=train.copy()<load_from_csv>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,
max_lighting=0.05, max_warp=0.)
%%time
print('{} Reading the 7 Gig 'test_v2.csv' will take ~4 minutes time! Go hydrate in the meanwhile :){}
'.format('
test = load_df_without_hits(path=dirname+'/test_v2.csv')
 print(list(test.columns))<feature_engineering>
data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train', ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder,bs=bs);
stats=data.batch_stats()
data.normalize(stats )
full_df=pd.concat([train,test],axis=0)
 _VALID_BROWSERS = [ 'Chrome', 'Safari', 'Firefox', 'Internet Explorer', 'Android Webview', 'Edge', 'Samsung Internet']
full_df['device.browser'].loc[~full_df['device.browser'].isin(_VALID_BROWSERS)]='Other'
_VALID_OPERATING_SYSTEMS = ['Windows', 'Macintosh', 'Android', 'iOS', 'Linux', 'Chrome OS', 'Windows Phone', 'Samsung']
full_df['device.operatingSystem'].loc[~full_df['device.operatingSystem'].isin(_VALID_OPERATING_SYSTEMS)]='Other'
full_df['date.weekday']=full_df['date'].dt.weekday
full_df['date.day']=full_df['date'].dt.day
full_df['date.month']=full_df['date'].dt.month
full_df['date.year']=full_df['date'].dt.year
print(full_df.nunique() [full_df.nunique() ==1].index)
 del(full_df['totals.bounces'])
 del(full_df['totals.newVisits'])
 del(full_df['socialEngagementType'])
 del(full_df['trafficSource.isTrueDirect'] )<sort_values>
data.show_batch(rows=5, figsize=(12,9))
full_df[['fullVisitorId','channelGrouping','date','visitNumber','totals.transactions','totals.transactionRevenue']][
full_df['fullVisitorId']=='9974232250427988367'].sort_values(by=['date','visitNumber'] )<groupby>
from sklearn.metrics import roc_auc_score
visitors_in_both_years= set(full_df['fullVisitorId'][full_df['date.year']==2016] ).intersection(set(full_df['fullVisitorId'][full_df['date.year']==2017]))
tmp_df=full_df[['fullVisitorId', 'totals.transactionRevenue']][full_df['fullVisitorId'].isin(visitors_in_both_years)]
print(tmp_df.groupby('fullVisitorId')['totals.transactionRevenue'].count() [(tmp_df.groupby('fullVisitorId')['totals.transactionRevenue'].count() >1)
 &(tmp_df.groupby('fullVisitorId')['totals.transactionRevenue'].sum() !=0)])
 del(tmp_df)
 del(visitors_in_both_years)
 del(full_df['visitId'] )<categorify>
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score
def standardize_traffic_source(val):
if 'google' in val:
return 'google'
elif 'youtube' in val:
return 'youtube'
elif 'facebook' in val:
return 'facebook'
elif 'yahoo' in val:
return 'yahoo'
elif 'baidu' in val:
return 'baidu'
elif 'aol' in val:
return 'aol'
elif 'quora' in val:
return 'quora'
elif 'reddit' in val:
return 'reddit'
elif 'pinterest' in val:
return 'pinterest'
elif 'bing' in val:
return 'bing'
elif 'wow' in val:
return 'wow'
elif 'blackboard' in val:
return 'blackboard'
elif len(val)>20:
return 'Other'
else:
return val
full_df['trafficSource.source']=full_df['trafficSource.source'].apply(standardize_traffic_source)
 categorical_variables=[
'trafficSource.medium','trafficSource.source']
fig, ax = plt.subplots(2,1, figsize=(16, 12))
plt.subplots_adjust(hspace=.5)
 for variable, subplot in zip(categorical_variables, ax.flatten()):
tmp=pd.pivot_table(data=full_df, index=variable, values='totals.transactionRevenue',
aggfunc='mean' ).reset_index().sort_values(by='totals.transactionRevenue', ascending=False)
 sns.barplot(data=tmp.head(50), x=variable, y='totals.transactionRevenue', ax=subplot ).set_xticklabels(subplot.get_xticklabels() ,rotation=90)
 subplot.set_title('Revenue vs {}'.format(variable.replace('trafficSource.','' ).capitalize()), fontdict={'fontweight':'bold'})
 del(tmp)
 plt.show() 
<categorify>
from torchvision.models import *
def standardize_traffic_keyword(keyword):
keyword=str(keyword ).lower()
if 'goo' in keyword or 'stor' in keyword:
return 'google'
elif 'yo' in keyword or 'yuotub' in keyword or 'yuo tub' in keyword or 'tub' in keyword or 'yutob' in keyword:
return 'youtube'
elif 'tshirt' in keyword or 't shirt' in keyword or 't-shirt' in keyword:
return 'tshirt'
elif 'shirts' in keyword or 'shirt' in keyword:
return 'shirts'
elif 'lamp' in keyword or 'lamps' in keyword:
return 'lamps'
elif 'android' in keyword:
return 'android'
elif 'merchandi' in keyword:
return 'merchandise'
elif bool(re.search(r'\d', keyword)) :
return 'other'
elif 'apparel' in keyword:
return 'apparel'
elif len(keyword)>28:
return 'other'
else:
return keyword
full_df['trafficSource.keyword']=full_df['trafficSource.keyword'].fillna('other')
 full_df['trafficSource.keyword']=full_df['trafficSource.keyword'].apply(standardize_traffic_keyword)
 plt.figure(figsize=(24,6))
keywordwise_revenue=pd.pivot_table(data=full_df, index=['trafficSource.keyword'], values='totals.transactionRevenue',
aggfunc='mean' ).sort_values(by='totals.transactionRevenue', ascending=False ).reset_index()
sns.barplot(data=keywordwise_revenue.head(50), x='trafficSource.keyword', y='totals.transactionRevenue', palette='Greens_r')
 plt.title('Revenue vs {}'.format('Keyword'), fontdict={'fontweight':'bold'})
 plt.xticks(rotation=90)
 plt.show()
del(keywordwise_revenue)
<feature_engineering>
learn = create_cnn(
 data,
densenet201,
path='.',
metrics=[auc_score],
ps=0.5
)
def preprocess_and_standardize(train_df, test_df):
full_df=pd.concat([train_df,test_df],axis=0)

full_df['date.weekday']=full_df['date'].dt.weekday
full_df['date.day']=full_df['date'].dt.day
full_df['date.month']=full_df['date'].dt.month
full_df['date.year']=full_df['date'].dt.year

 print('Columns with cardinality=1 ie Constant value : ',full_df.nunique() [full_df.nunique() ==1].index)
 _COLS_TO_DROP=[ 'trafficSource.adContent', 'visitId', 'device.isMobile', 'trafficSource.referralPath',
'totals.totalTransactionRevenue', 'visitNumber', 'visitStartTime',
'totals.bounces','totals.newVisits','socialEngagementType','trafficSource.isTrueDirect',]
for c in _COLS_TO_DROP:
try:
del(full_df[c])
 except:
pass
print('{} I)DROPPED UNNECESSARY COLUMNS {}'.format('

 _VALID_BROWSERS = [ 'Chrome', 'Safari', 'Firefox', 'Internet Explorer', 'Android Webview', 'Edge', 'Samsung Internet']
full_df['device.browser'].loc[~full_df['device.browser'].isin(_VALID_BROWSERS)]='Other'

 _VALID_OPERATING_SYSTEMS = ['Windows', 'Macintosh', 'Android', 'iOS', 'Linux', 'Chrome OS', 'Windows Phone', 'Samsung']
full_df['device.operatingSystem'].loc[~full_df['device.operatingSystem'].isin(_VALID_OPERATING_SYSTEMS)]='Other'

 full_df['trafficSource.keyword']=full_df['trafficSource.keyword'].fillna('other')
 full_df['trafficSource.keyword']=full_df['trafficSource.keyword'].apply(standardize_traffic_keyword)
 full_df['trafficSource.source']=full_df['trafficSource.source'].apply(standardize_traffic_source)
 full_df['trafficSource.adwordsClickInfo.slot']=full_df['trafficSource.adwordsClickInfo.slot'].fillna('No Traffic-ad Slot')
 print('{} II)STANDARDIZED COLUMN VALUES {}'.format('

 _CAT_COLS_FOR_ENCODER = ['channelGrouping', 'device.browser', 'device.deviceCategory', 'device.operatingSystem',
'geoNetwork.city','geoNetwork.continent', 'geoNetwork.country', 'geoNetwork.metro', 'geoNetwork.region',
'geoNetwork.subContinent','geoNetwork.networkDomain','trafficSource.medium', 'trafficSource.source',
'trafficSource.campaign','trafficSource.keyword','trafficSource.adwordsClickInfo.slot']
print('{} III)ENCODING RELEVANT COLUMNS TO LABELS {}'.format('
for col in _CAT_COLS_FOR_ENCODER:
print('Converting <',col,'> to labels.')
 if full_df[col].dtype.name=='object':
lb = LabelEncoder()
full_df[col]=lb.fit_transform(full_df[col])
 return full_df
def return_aggregated_df(input_df, i=0, create_test=False, verbose=True):
if create_test==False:

 feature_df = input_df[(pd.to_datetime(input_df['date'])> time_from.index[i])&(pd.to_datetime(input_df['date'])<= time_end.index[i])].copy()
else:

 feature_df = input_df
if verbose==True:
print('{} A)CREATED BASE-FEATURE DATAFRAME {}'.format('
_COLS_TO_MEDIAN_VAL = ['channelGrouping' , 'device.browser' , 'device.deviceCategory' , 'device.operatingSystem' , 'geoNetwork.city' , 'geoNetwork.country' ,
'geoNetwork.metro' , 'geoNetwork.networkDomain' , 'geoNetwork.region' , 'geoNetwork.subContinent', 'geoNetwork.continent' ,
'trafficSource.medium' , 'trafficSource.source' , 'trafficSource.campaign' , 'trafficSource.keyword' , 'trafficSource.adwordsClickInfo.slot']
_COLS_TO_SUM_VAL = ['totals.hits' , 'totals.sessionQualityDim' , 'totals.transactions', 'totals.pageviews' ]
_COLS_TO_MEAN_VAL = ['date.year', 'date.month', 'date.day', 'date.weekday']
_COLS_TO_STD_DEV_VAL = ['totals.hits' , 'totals.pageviews' , 'totals.sessionQualityDim']
ans=pd.concat([
feature_df.groupby('fullVisitorId')[_COLS_TO_MEDIAN_VAL].median().add_suffix('_median'),
feature_df.groupby('fullVisitorId')[_COLS_TO_MEAN_VAL].mean().add_suffix('_mean'),
feature_df.groupby('fullVisitorId')[_COLS_TO_SUM_VAL].sum().add_suffix('_sum'),
feature_df.groupby('fullVisitorId')[_COLS_TO_STD_DEV_VAL].std(ddof=0 ).add_suffix('_stdDev'),
feature_df.groupby('fullVisitorId')['date'].apply(lambda x:(x.max() -x.min())//10**9 ).rename('time_diff')
 ], axis=1 ).reset_index()
if verbose==True:
print('{} B)AGGREGATED THE BASE FEATURES {}'.format('
if create_test==False:

 target_df = input_df[(input_df['date'] > pd.Timestamp(time_from.values[i]))
&(input_df['date'] <= pd.Timestamp(time_from.values[i+1])) ].copy()
ans=ans.merge(target_df.groupby('fullVisitorId')['totals.transactionRevenue'].sum().reset_index() , how = 'left', on = 'fullVisitorId')
 if verbose==True:
print('{} C)MERGED THE FUTURE TIMEFRAME TARGET WITH AGGREGATED FEATURES {}'.format('
ans['totals.transactionRevenue']=ans['totals.transactionRevenue'].fillna(0 ).apply(lambda y: np.log(y)if y>0 else y)
 return ans<data_type_conversions>
lr = 2e-03
submissions_df=pd.DataFrame()
test_df = full_df.iloc[train_shape[0]:, :]
test_x=return_aggregated_df(test_df,i=0,create_test=True)
 submissions_df['fullVisitorId']=test_x['fullVisitorId'].copy()
test_x=test_x.drop(['fullVisitorId'],axis=1 ).astype(int)
 print('{} IV)CREATED TEST-DATASET FOR APPROACH 2 {}'.format('<prepare_x_and_y>
learn.fit_one_cycle(1,lr)
 learn.recorder.plot()
learn.recorder.plot_losses()
try:
del(test_df)
 del(tmp)
 del(val_df)
 except:
pass
submission_catboost_preds = np.zeros(len(test_x))
submission_xgb_preds = np.zeros(len(test_x))
submission_lgb_preds = np.zeros(len(test_x))
kfold=list(range(time_from.shape[0]-1))
kfold.remove(4)
 for fold in kfold:
fold_df = return_aggregated_df(full_df, fold, verbose=False)
 train_y = fold_df['totals.transactionRevenue']
train_x = fold_df.copy().drop(['fullVisitorId','totals.transactionRevenue'],inplace=False,axis=1 ).astype(int)
 print('
{} {}.1)CREATED TRAIN SET OF <{}> FOR THIS FOLD {}'.format(26*'
try:
del(fold_df)
 except:
pass
print('
{} {}.2)TRAINING CatBoostRegressor FOR THIS FOLD {}'.format(26*'
catboost = CatBoostRegressor(iterations=1100,learning_rate=0.04, depth=10, bagging_temperature = 0.2, od_wait=20,
eval_metric='RMSE', random_seed=0, od_type='Iter')
 catboost.fit(train_x , train_y, eval_set=[(validation_x, validation_y)],early_stopping_rounds=100, use_best_model=True, verbose=100)
 predictions = catboost.predict(validation_x)
 predictions[predictions<0]=0
print('{}{}.3)CATBOOST REGRESSION LOSS = {}{}
'.format('>'*26, fold, mean_squared_error(validation_y, predictions)**.5, '<'*26))
submission_preds_raw=catboost.predict(test_x)
 submission_preds_raw[submission_preds_raw<0]=0
submission_catboost_preds += submission_preds_raw / len(kfold)
 print('
{} {}.4)TRAINING XGBRegressor FOR THIS FOLD {}'.format(26*'
xgb = XGBRegressor(n_estimators=1000, max_depth=20, learning_rate=0.07, booster='gbtree', random_state=0, importance_type='total_gain',
objective= 'reg:squarederror', gamma=1.45, alpha=0.0, subsample=0.67, colsample_bytree=0.054, colsample_bylevel= 0.50)
 xgb.fit(train_x , train_y ,eval_set=[(validation_x, validation_y)], early_stopping_rounds=100, eval_metric='rmse', verbose=100)
 predictions = xgb.predict(validation_x)
 predictions[predictions<0]=0
print('{}{}.5)XGB LOSS = {}{}
'.format('>'*26, fold, mean_squared_error(validation_y, predictions)**.5, '<'*26))
submission_preds_raw=xgb.predict(test_x)
 submission_preds_raw[submission_preds_raw<0]=0
submission_xgb_preds += submission_preds_raw / len(kfold)
 print('
{} {}.6)TRAINING LGBRegressor FOR THIS FOLD {}'.format(26*'
lgb_reg = lgb.LGBMRegressor(n_estimators=1100, objective='regression', max_depth=-1, learning_rate=0.08, min_child_samples=40, boosting='gbdt',
feature_fraction=0.9, bagging_freq=10, bagging_fraction=0.8, bagging_seed=3, metric='rmse', lambda_l1=0, lambda_l2=1, verbosity=100)
 lgb_reg.fit(train_x, train_y, eval_set= [(validation_x, validation_y)], eval_metric='rmse', verbose=100 ,early_stopping_rounds = 100)
 predictions = lgb_reg.predict(validation_x.values, num_iteration=lgb_reg.best_iteration_)
 predictions[predictions<0]=0
print('{}{}.7)LGB LOSS = {}{}
'.format('>'*26, fold, mean_squared_error(validation_y.values, predictions)**.5, '<'*26))
submission_preds_raw=lgb_reg.predict(test_x.values, num_iteration=lgb_reg.best_iteration_)
 submission_preds_raw[submission_preds_raw<0]=0
submission_lgb_preds += submission_preds_raw / len(kfold )<save_to_csv>
learn.unfreeze()
learn.lr_find()
submissions_df['PredictedLogRevenue'] = submission_lgb_preds*0.4 + submission_catboost_preds*0.5 + submission_xgb_preds*0.1
submissions_df.to_csv('result.csv', index=False)
 print("Done" )<load_from_csv>
learn.fit_one_cycle(1,slice(1e-4,1e-3))
def load_df(csv_path, nrows=None):
USE_COLUMNS = [
'channelGrouping', 'date', 'device', 'fullVisitorId', 'geoNetwork',
'socialEngagementType', 'totals', 'trafficSource', 'visitId',
'visitNumber', 'visitStartTime', 'customDimensions'
]
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
df = pd.read_csv(csv_path,
converters={column: json.loads for column in JSON_COLUMNS},
dtype={'fullVisitorId': 'str'},
nrows=nrows, usecols=USE_COLUMNS)
 for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 print(f"Loaded {os.path.basename(csv_path)}.Shape: {df.shape}")
 return df<load_from_csv>
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score
%%time
train_df = load_df(".. /input/train_v2.csv")
 test_df = load_df(".. /input/test_v2.csv" )<drop_column>
preds,y=learn.TTA()
pred_score_tta=auc_score(preds,y)
 pred_score_tta
train_df = train_df.drop(labels=['trafficSource.campaignCode'],axis=1)
 train_df = train_df.drop(labels=['customDimensions','totals.totalTransactionRevenue'],axis=1)
 test_df = test_df.drop(labels=['customDimensions','totals.transactionRevenue','totals.totalTransactionRevenue'],axis=1 )<data_type_conversions>
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )
train_df['date']= pd.to_datetime(train_df['date'],format='%Y%m%d')
 test_df['date']= pd.to_datetime(test_df['date'],format='%Y%m%d')
 train_df.head()<data_type_conversions>
preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test )
train_df['totals.transactionRevenue'] = train_df['totals.transactionRevenue'].astype('float')
 print('Total records: ', len(train_df), 'customers doesnt contribute for revenue: ',train_df['totals.transactionRevenue'].isna().sum() , \
'customers contributing revenue: ',len(train_df)- train_df['totals.transactionRevenue'].isna().sum() )<count_unique_values>
sub=pd.read_csv(f'{path}/sample_submission.csv' ).set_index('id')
 sub.head()
print('Features with one unique values are :')
 print(train_df.columns[train_df.nunique() == 1])
 new_df = train_df.drop(labels = train_df.columns[train_df.nunique() == 1],
axis = 1)
 test_df = test_df.drop(labels = train_df.columns[train_df.nunique() == 1],
axis = 1 )<sort_values>
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )
geofields = ['geoNetwork.city','geoNetwork.continent',
'geoNetwork.country','geoNetwork.metro',
'geoNetwork.networkDomain','geoNetwork.region',
'geoNetwork.subContinent']
for fields in geofields:
temp_df = new_df.groupby(by=fields ).size().sort_values(ascending=False ).head(10)
 print(temp_df)
 print('*'*30 )<count_missing_values>
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'submission_{pred_score}.csv' )
<drop_column><EOS> 
sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])
 sub.to_csv(f'submission_{pred_score_tta}.csv' )