Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<compute_train_metric>
warnings.simplefilter('ignore', category=FutureWarning)
 warnings.simplefilter('ignore', category=UserWarning)
 %matplotlib inline
lr = LogisticRegression(solver="lbfgs",max_iter=500,n_jobs=4)
 lengte=100000
lr.fit(train[:lengte], labels[:lengte])
 lr_pred = lr.predict_proba(train[:lengte])[:, 1]
score = roc_auc_score(labels[:lengte], lr_pred)
 print("score: ", score)
 lr_pred = lr.predict(train[:lengte])
 score = classification_report(labels[:lengte], lr_pred)
 print("score: ", score)
 print("First Random float number: ", random.random())
 lr_pred = [(random.random() >0.7)*1 for x in range(lengte)]
score = classification_report(labels[:lengte], lr_pred)
 print("score: ", score)
<predict_on_test>
train_df = pd.read_csv(PATH/'train_labels.csv')
 print(train_df.shape)
 train_df.head()
submission["id"] = test_id
submission["target"] = lr.predict_proba(test)[:, 1]<save_to_csv>
train_df['label'].value_counts(normalize=True )
submission.to_csv("submission.csv", index=False )<save_to_csv>
src =(ImageItemList.from_csv(PATH, folder='train', csv_name='train_labels.csv', suffix='.tif')
.random_split_by_pct(0.1, seed=77)
.label_from_df() 
.add_test_folder() )
submission.to_csv("submission.csv", index=False )<define_variables>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0, max_zoom=1., max_lighting=0.05, max_warp=0)
 data =(src.transform(tfms, size=96, resize_method=ResizeMethod.SQUISH)
.databunch(bs=64, path='.'))
data.normalize(imagenet_stats);
def to_na(df):
to_NA_cols = ['trafficSource_adContent','trafficSource_adwordsClickInfo.adNetworkType',
'trafficSource_adwordsClickInfo.slot','trafficSource_adwordsClickInfo.gclId',
'trafficSource_keyword','trafficSource_referralPath','customDimensions_value']
to_0_cols = ['totals_transactionRevenue','trafficSource_adwordsClickInfo.page','totals_sessionQualityDim','totals_bounces',
'totals_timeOnSite','totals_newVisits','totals_pageviews','customDimensions_index','totals_transactions','totals_totalTransactionRevenue']
to_true_cols = ['trafficSource_adwordsClickInfo.isVideoAd']
to_false_cols = ['trafficSource_isTrueDirect']
df[to_NA_cols] = df[to_NA_cols].fillna('NA')
 df[to_0_cols] = df[to_0_cols].fillna(0)
 df[to_true_cols] = df[to_true_cols].fillna(True)
 df[to_false_cols] = df[to_false_cols].fillna(False)
 return df
def encode_date(df):
fld = pd.to_datetime(df['date'], infer_datetime_format=True)
 attrs = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear',
'Is_month_end', 'Is_month_start', 'Is_quarter_end',
'Is_quarter_start', 'Is_year_end', 'Is_year_start','Hour']
for attr in attrs:
df['Date_'+attr] = getattr(fld.dt,attr.lower())
 return df
def weird_na(df):
cols_to_replace = {
'socialEngagementType' : 'Not Socially Engaged',
'device_browserSize' : 'not available in demo dataset',
'device_flashVersion' : 'not available in demo dataset',
'device_browserVersion' : 'not available in demo dataset',
'device_language' : 'not available in demo dataset',
'device_mobileDeviceBranding' : 'not available in demo dataset',
'device_mobileDeviceInfo' : 'not available in demo dataset',
'device_mobileDeviceMarketingName' : 'not available in demo dataset',
'device_mobileDeviceModel' : 'not available in demo dataset',
'device_mobileInputSelector' : 'not available in demo dataset',
'device_operatingSystemVersion' : 'not available in demo dataset',
'device_screenColors' : 'not available in demo dataset',
'device_screenResolution' : 'not available in demo dataset',
'geoNetwork_city' : 'not available in demo dataset',
'geoNetwork_cityId' : 'not available in demo dataset',
'geoNetwork_latitude' : 'not available in demo dataset',
'geoNetwork_longitude' : 'not available in demo dataset',
'geoNetwork_metro' : ['not available in demo dataset', '(not set)'],
'geoNetwork_networkDomain' : 'unknown.unknown',
'geoNetwork_networkLocation' : 'not available in demo dataset',
'geoNetwork_region' : 'not available in demo dataset',
'trafficSource_adwordsClickInfo.criteriaParameters' : 'not available in demo dataset',
'trafficSource_campaign' : '(not set)',
'trafficSource_keyword' : '(not provided)',
'networkDomain': '(not set)',
'city': '(not set)',
}
df = df.replace(cols_to_replace,'NA')
 return df
def del_const(df):
const_col = []
for col in df.columns:
if df[col].nunique() == 1 and df[col].isnull().sum() ==0 :
const_col.append(col)
 df.drop(const_col,axis=1,inplace=True)
 return df, const_col
def json_it(df):
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}_{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 df['customDimensions']=df['customDimensions'].apply(literal_eval)
 df['customDimensions']=df['customDimensions'].str[0]
df['customDimensions']=df['customDimensions'].apply(lambda x: {'index':np.NaN,'value':np.NaN} if pd.isnull(x)else x)
 column_as_df = json_normalize(df['customDimensions'])
 column_as_df.columns = [f"customDimensions_{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop('customDimensions', axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 return df
def convert_it(df):
df = weird_na(df)
 df = to_na(df)
 df_train = encode_date(df)
 return df
def fix_type(df):
try:
df.drop('trafficSource_campaignCode',axis=1,inplace=True)
 except:
pass
df['totals_transactionRevenue'] = df['totals_transactionRevenue'].fillna(0 ).astype(float)
 to_int = ['totals_bounces','totals_newVisits','totals_pageviews',
'customDimensions_index','totals_hits','totals_sessionQualityDim',
'totals_visits','totals_timeOnSite','trafficSource_adwordsClickInfo.page',
'totals_transactions','totals_totalTransactionRevenue']
for col in to_int :
df[col] = df[col].astype(int)
 return df
def load_it(csv_path,name):
CONST_COLLUMNS = ['socialEngagementType','device_browserSize',
'device_browserVersion','device_flashVersion',
'device_language','device_mobileDeviceBranding',
'device_mobileDeviceInfo','device_mobileDeviceMarketingName',
'device_mobileDeviceModel','device_mobileInputSelector',
'device_operatingSystemVersion','device_screenColors',
'device_screenResolution','geoNetwork_cityId',
'geoNetwork_latitude','geoNetwork_longitude',
'geoNetwork_networkLocation',
'trafficSource_adwordsClickInfo.criteriaParameters',]
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
dfs = pd.read_csv(csv_path, sep=',',
parse_dates=['date'],
converters={column: json.loads for column in JSON_COLUMNS},
dtype={'fullVisitorId': 'str'},
chunksize = 200000)
 for idx,df in enumerate(dfs):
print(idx)
 df.reset_index(drop = True,inplace = True)
 df = json_it(df)
 df = convert_it(df)
 df.drop(CONST_COLLUMNS,axis=1,inplace=True)
 df.drop('hits',axis=1,inplace=True)
 df = fix_type(df)
 df.to_pickle(f'{name}_{idx}.pkl')
 del df
gc.collect()
print('Done')
 load_it('.. /input/ga-customer-revenue-prediction/train_v2.csv','train')
 load_it('.. /input/ga-customer-revenue-prediction/test_v2.csv','test' )<set_options>
data.show_batch(rows=5, figsize=(15, 15))
data_root = './'
print(os.listdir(data_root))
pd.set_option('display.max_rows',200)
<categorify>
class FocalLoss(nn.Module):
def __init__(self, alpha=1., gamma=1.) :
super().__init__()
self.alpha = alpha
self.gamma = gamma
def forward(self, inputs, targets, **kwargs):
CE_loss = nn.CrossEntropyLoss(reduction='none' )(inputs, targets)
 pt = torch.exp(-CE_loss)
 F_loss = self.alpha *(( 1-pt)**self.gamma)* CE_loss
return F_loss.mean()
def roc_score(inp, target):
_, indices = inp.max(1)
 return torch.Tensor([roc_auc_score(target, indices)])[0]
def load_data(data='train',n=2):
df = pd.DataFrame()
for i in range(n):
if data=='train':
if i > 8 :
break
dfpart = pd.read_pickle(pjoin(data_root,f'train_{i}.pkl'))
elif data=='test':
if i > 2 :
break
dfpart = pd.read_pickle(pjoin(data_root,f'test_{i}.pkl'))
df = pd.concat([df,dfpart])
 del dfpart
return df 
<load_from_csv>
loss_func = FocalLoss(gamma=1.)
learn = create_cnn(data, models.densenet121, metrics=[accuracy, roc_score], loss_func=loss_func )
df_train = load_data(n=9)
 df_test = load_data('test',n=4 )<concatenate>
learn.lr_find()
df = pd.concat([df_train, df_test] )<define_variables>
learn.fit_one_cycle(2, 1e-3 )
col_drop = ['Date_Year', 'Date_Month', 'Date_Week','Date_Hour','device_isMobile','device_deviceCategory',
'Date_Day', 'Date_Dayofweek', 'Date_Dayofyear', 'Date_Is_month_end',
'Date_Is_month_start', 'Date_Is_quarter_end', 'Date_Is_quarter_start',
'Date_Is_year_end', 'Date_Is_year_start','totals_visits',
'date','visitId','totals_totalTransactionRevenue','geoNetwork_city','geoNetwork_continent',
'geoNetwork_metro','geoNetwork_networkDomain',
'geoNetwork_region','geoNetwork_subContinent','trafficSource_adContent',
'trafficSource_adwordsClickInfo.adNetworkType','trafficSource_adwordsClickInfo.gclId',
'trafficSource_adwordsClickInfo.slot','trafficSource_campaign',
'trafficSource_keyword','trafficSource_referralPath','trafficSource_medium',
'customDimensions_value','customDimensions_index','trafficSource_source',
'totals_bounces','visitNumber','totals_newVisits']
df.drop(col_drop, axis=1, inplace=True )<feature_engineering>
learn.save('stage-1' )
country_drop=df.groupby('geoNetwork_country')['totals_transactions'].sum() [df.groupby('geoNetwork_country')['totals_transactions'].sum().sort_values() <4].index.tolist()
df.loc[df[df.geoNetwork_country.isin(country_drop)].index,'geoNetwork_country'] = 'NaN'
df.loc[df[~df.device_browser.isin(['Edge', 'Internet Explorer', 'Firefox', 'Safari', 'Chrome'])].index,'device_browser'] = 'NaN'
df.loc[df[~df.device_operatingSystem.isin(['Android', 'iOS', 'Linux', 'Chrome OS', 'Windows', 'Macintosh'])].index,'device_operatingSystem'] = 'NaN'<categorify>
learn.load('stage-1');
col_lb = ['channelGrouping','device_browser','device_operatingSystem', 'geoNetwork_country',
'trafficSource_adwordsClickInfo.isVideoAd','trafficSource_isTrueDirect']
for col in col_lb:
lb = LabelEncoder()
df[col]=lb.fit_transform(df[col] )<define_variables>
learn.lr_find()
to_median = ['channelGrouping','device_browser','device_operatingSystem','geoNetwork_country','trafficSource_adwordsClickInfo.isVideoAd','trafficSource_isTrueDirect','trafficSource_adwordsClickInfo.page']
to_sum =['totals_hits','totals_pageviews','totals_timeOnSite','totals_transactionRevenue', 'totals_transactions']
to_mean =['totals_hits','totals_pageviews','totals_sessionQualityDim']
to_std = ['totals_hits','totals_pageviews']
to_time = 'visitStartTime'<data_type_conversions>
max_lr = 1e-4
learn.fit_one_cycle(4, slice(1e-6, max_lr))
target_period = pd.date_range(start='2016-08-01',end='2018-12-01', freq='2MS')
 train_period = target_period.to_series().shift(periods=-210, freq='d',axis= 0)
 time_to = train_period[train_period.index>np.datetime64('2016-08-01')].astype('int')//10**9
time_end = target_period.to_series().shift(periods=-45, freq='d',axis= 0)[4:]<groupby>
learn.save('stage-2' )
user_x = df.iloc[df_train.shape[0]:,:]
test_x = pd.concat([user_x.groupby('fullVisitorId')[to_median].median().add_suffix('_median'),
user_x.groupby('fullVisitorId')['visitStartTime'].agg(['first','last'] ).add_suffix('_time' ).sub(time_to.values[-1] ).abs() ,
user_x.groupby('fullVisitorId')['visitStartTime'].apply(lambda x: x.max() -x.min() ).rename('time_diff'),
user_x.groupby('fullVisitorId')[to_sum].sum().add_suffix('_sum'),
user_x.groupby('fullVisitorId')[to_mean].mean().add_suffix('_mean'),
user_x.groupby('fullVisitorId')[to_std].std(ddof=0 ).add_suffix('_std')],axis=1 ).reset_index()
test_ID= test_x.fullVisitorId
test_x = test_x.drop(['fullVisitorId'], axis=1,errors='ignore')
 test_x = test_x.astype('int' )<prepare_x_and_y>
learn.load('stage-2');
i=4
user_x = df[(df.visitStartTime>=(time_to.index.astype('int')//10**9)[i])&(df.visitStartTime<(time_end.index.astype('int')//10**9)[i])]
user_y = df[(df.visitStartTime>=time_to.values[i])&(df.visitStartTime<time_to.values[i+1])]
train_x = pd.concat([user_x.groupby('fullVisitorId')[to_median].median().add_suffix('_median'),
user_x.groupby('fullVisitorId')['visitStartTime'].agg(['first','last'] ).add_suffix('_time' ).sub(time_to.values[i] ).abs() ,
user_x.groupby('fullVisitorId')['visitStartTime'].apply(lambda x: x.max() -x.min() ).rename('time_diff'),
user_x.groupby('fullVisitorId')[to_sum].sum().add_suffix('_sum'),
user_x.groupby('fullVisitorId')[to_mean].mean().add_suffix('_mean'),
user_x.groupby('fullVisitorId')[to_std].std(ddof=0 ).add_suffix('_std')],axis=1 ).reset_index()
merged=train_x.merge(user_y.groupby('fullVisitorId')['totals_transactionRevenue'].sum().reset_index() ,\
how='left', on='fullVisitorId')
 val_y = merged.totals_transactionRevenue
val_y.fillna(0, inplace=True)
 val_x = merged.drop(['fullVisitorId','totals_transactionRevenue'], axis=1,errors='ignore')
 val_x = val_x.astype('int' )<import_modules>
preds, y = learn.TTA(beta=0.4, ds_type=DatasetType.Test)
 preds = torch.softmax(preds, dim=1)[:, 1].numpy()
import lightgbm as lgb
from xgboost import XGBRegressor
from catboost import CatBoostRegressor
from sklearn.metrics import mean_squared_error<init_hyperparams>
test_ids = [f.stem for f in learn.data.test_ds.items]
subm = pd.read_csv(PATH/'sample_submission.csv')
 orig_ids = list(subm['id'] )
params={'learning_rate': 0.01,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
'bagging_fraction': 0.9,
'feature_fraction': 0.9,
"random_state":42,
'max_depth': 5,
"bagging_seed" : 42,
"verbosity" : -1,
"bagging_frequency" : 5,
'lambda_l2': 0.5,
'lambda_l1': 0.5,
'min_child_samples': 36
}
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.02,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456,
'importance_type': 'total_gain'
}
cat_param = {
'learning_rate' :0.03,
'depth' :10,
'eval_metric' :'RMSE',
'od_type' :'Iter',
'metric_period ' : 50,
'od_wait' : 20,
'seed' : 42
}<prepare_x_and_y>
def create_submission(orig_ids, test_ids, preds):
preds_dict = dict(( k, v)for k, v in zip(test_ids, preds))
pred_cor = [preds_dict[id] for id in orig_ids]
df = pd.DataFrame({'id':orig_ids,'label':pred_cor})
 df.to_csv(f'submission_{auc_val}.csv', header=True, index=False)
 return df
<save_to_csv><EOS> 
test_df = create_submission(orig_ids, test_ids, preds)
 test_df.head()