Kaggle competition: Predicting Red Hat Business Value
<SOS> metric: AUC Kaggle data source: predicting-red-hat-business-value<define_variables>
%matplotlib inline
sns.set()
print(os.listdir(".. /input"))
warnings.filterwarnings("ignore" )
base = ".. /input/siim-isic-melanoma-classification/"
models = ".. /input/pytorch-pretrained-image-models/"
imagestats = ".. /input/siimisic-melanoma-classification-image-stats/"<load_from_csv>
train=pd.read_csv('.. /input/act_train.csv',parse_dates=['date'])
 test=pd.read_csv('.. /input/act_test.csv',parse_dates=['date'])
 people=pd.read_csv('.. /input/people.csv',parse_dates=['date'] )
train_data = pd.read_csv(base + "train.csv")
 train_data.head()<count_missing_values>
train=pd.merge(train,people,on='people_id')
 test=pd.merge(test,people,on='people_id' )
100*train_data.isnull().mean()<count_unique_values>
train.char_38.nunique()
train_data.nunique()<load_from_csv>
len(set(test['group_1'])-set(train['group_1']))
test_data = pd.read_csv(base + "test.csv")
 test_data.head()<install_modules>
for i in train.columns:
if np.dtype(train[i])==np.dtype('object'):
train[i].fillna('type 0',inplace=True )
!pip install chart_studio 
<count_values>
checkMissing(train)
 checkMissing(test )
print("Total images in Train set: ",train_data['image_name'].count())
 print("Total images in Test set: ",test_data['image_name'].count() )<groupby>
outcomeMeanGroupbyGroup_1=train.groupby(['group_1'])['outcome'].mean().to_frame().reset_index()
outcomeMedianGroupbyGroup_1=train.groupby(['group_1'])['outcome'].median().to_frame().reset_index()
dict_outcomeMeanGroupbyGroup_1=dict(zip(outcomeMeanGroupbyGroup_1['group_1'],outcomeMeanGroupbyGroup_1['outcome']))
dict_outcomeMedianGroupbyGroup_1=dict(zip(outcomeMedianGroupbyGroup_1['group_1'],outcomeMedianGroupbyGroup_1['outcome']))
train['outcomeMeanGroupbyGroup_1']=train['group_1'].map(lambda x: dict_outcomeMeanGroupbyGroup_1.get(x))
train['outcomeMedianGroupbyGroup_1']=train['group_1'].map(lambda x: dict_outcomeMedianGroupbyGroup_1.get(x))
test['outcomeMeanGroupbyGroup_1']=test['group_1'].map(lambda x: dict_outcomeMeanGroupbyGroup_1.get(x))
test['outcomeMedianGroupbyGroup_1']=test['group_1'].map(lambda x: dict_outcomeMedianGroupbyGroup_1.get(x))
groupped_data = train_data.groupby(['benign_malignant', 'sex'] ).agg({'target': 'count'})
 groupped_data['%'] = groupped_data.groupby(level=0 ).apply(lambda x: 100*x/x.sum())
 groupped_data<groupby>
checkMissing(test )
groupped_data = train_data.groupby(['diagnosis'] ).agg({'age_approx': 'mean'})
 groupped_data<groupby>
test['outcomeMeanGroupbyGroup_1']=test.groupby(['activity_category'])['outcomeMeanGroupbyGroup_1'].transform(lambda x: x.fillna(x.mean()))
test['outcomeMedianGroupbyGroup_1']=test.groupby(['activity_category'])['outcomeMedianGroupbyGroup_1'].transform(lambda x: x.fillna(x.mean()))
groupped_data = train_data.groupby(['diagnosis', 'anatom_site_general_challenge'] ).agg({'target': 'count'})
 groupped_data['%'] = groupped_data.groupby(level=0 ).apply(lambda x: 100*x/x.sum())
 groupped_data = groupped_data.drop(['target'], axis=1)
 groupped_data = groupped_data.reset_index()<groupby>
checkMissing(test )
ids = data['patient_id'].unique().tolist()
l = []
k = 0
for i, id in enumerate(ids):
d = data.loc[data['patient_id'] == id, ].copy()
d = d.reset_index(drop=True)
 index = d.loc[d['target'] == 1, ].index.values[0]
if index == 0:
k = k+1
continue
else:
d = d[d.index < index]
groupped_data = d.groupby(['benign_malignant', 'diagnosis'] ).agg({'target': 'count'})
 groupped_data['%'] = groupped_data.groupby(level=0 ).apply(lambda x: 100*x/x.sum())
 groupped_data = groupped_data.reset_index()
l.append(groupped_data)
 print(k)
 print(len(l))<groupby>
def featureEngineering(df):
listDate=['year', 'month', 'week', 'day', 'dayofweek', 'dayofyear','is_month_end', 'is_month_start', 'is_quarter_end', 'is_quarter_start', 'is_year_end', 'is_year_start']
for n in listDate:
df[n.upper() ]=df['date_x'].map(lambda x: getattr(x,n))
df[n.upper() ]=df['date_y'].map(lambda x: getattr(x,n))
for i in df.columns:
if np.dtype(df[i])==np.dtype('object')and i not in ['activity_id','people_id']:
df[i]=df[i].map(lambda x:int(x.split(' ')[1]))
elif np.dtype(df[i])==np.dtype('bool'):
df[i]=df[i].map(lambda x:int(x))
return df
groupped_data = train_data.groupby(['sex', 'diagnosis'] ).agg({'target': 'count'})
 groupped_data['%'] = groupped_data.groupby(level=0 ).apply(lambda x: 100*x/x.sum())
 groupped_data<groupby>
X_train=train.drop(['outcome','date_x','date_y','activity_id','people_id'],axis=1 ).copy()
y_train=train['outcome'].copy()
X_test=test.drop(['date_x','date_y','activity_id','people_id'],axis=1 ).copy()
groupped_data = train_data.groupby(['sex', 'diagnosis'] ).agg({'age_approx': 'mean'})
 groupped_data<groupby>
columns = ['benign_malignant', 'sex', 'target']
train_data[columns].groupby(['benign_malignant', 'sex'] ).count()<define_variables>
from lightgbm import LGBMClassifier
from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold
from sklearn.metrics import roc_auc_score
basepath = ".. /input/siim-isic-melanoma-classification/"
modelspath = ".. /input/pytorch-pretrained-image-models/"
imagestatspath = ".. /input/siimisic-melanoma-classification-image-stats/"<define_variables>
kfold=StratifiedKFold(n_splits=2 )
example_files = os.listdir(basepath + "train/")[0:2]
example_files<load_from_csv>
lgbm = LGBMClassifier(random_state=8)
 lgbm_param_grid = {'num_leaves' : [2,3],
'learning_rate' : [0.004,0.005,0.006,0.007],
'n_estimators': [50,100]}
lgbm = GridSearchCV(lgbm,param_grid = lgbm_param_grid, cv=kfold, scoring="accuracy",n_jobs=2, verbose = 1)
 lgbm.fit(X_train,y_train)
 print(lgbm.best_score_)
 print(lgbm.best_params_ )
test_image_stats = pd.read_csv(imagestatspath + "test_image_stats.csv")
 test_image_stats.head(1 )<define_variables>
results = pd.DataFrame({ 'activity_id' : test['activity_id'].values,
'outcome': lgbm.predict_proba(X_test)[:,1] } )
<install_modules><EOS> 
results.to_csv('red_hat_LightGBM.csv',index=False )