Kaggle competition: Summer Analytics 2020 Capstone Project
<SOS> metric: auc Kaggle data source: summer-analytics-2020-capstone-project<count_values>
%matplotlib inline
plt.style.use('seaborn-whitegrid')
 warnings.filterwarnings('ignore' )
train_data['Color'].value_counts()<count_values>
df=pd.read_csv('.. /input/summeranalytics2020/train.csv')
 df.shape
test_data['Color'].value_counts()<data_type_conversions>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 df.head()
train_data['Color'].fillna(value='Color_Unknown',inplace=True)
 test_data['Color'].fillna(value='Color_Unknown',inplace=True )<count_missing_values>
test_df=pd.read_csv('.. /input/summeranalytics2020/test.csv')
 print(test_df.shape )
print("Number of null values in Color column "+str(train_data['Color'].isnull().sum()))
print("Number of null values in Color column "+str(test_data['Color'].isnull().sum()))<count_values>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 test_df.head()
train_data['Transmission'].value_counts()<count_values>
df_combine=pd.concat([df,test_df],axis=0, sort = False,ignore_index = True)
 df_combine.shape
test_data['Transmission'].value_counts()<filter>
ordinal_catg_col = ["Education", "EnvironmentSatisfaction", "JobInvolvement","JobSatisfaction",
"PerformanceRating","StockOptionLevel","CommunicationSkill", "Behaviour" ]
print(ordinal_catg_col )
train_data[train_data['Transmission']=='Manual']<rename_columns>
df_BusinessTravel_one_hot = pd.get_dummies(df_combine['BusinessTravel'], prefix='BusinessTravel')
 df_Department_one_hot = pd.get_dummies(df_combine['Department'], prefix='Department')
 df_EducationField_one_hot = pd.get_dummies(df_combine['EducationField'], prefix='EducationField')
 df_Gender_one_hot = pd.get_dummies(df_combine['Gender'], prefix='Gender',drop_first=True)
 df_JobRole_one_hot = pd.get_dummies(df_combine['JobRole'], prefix='JobRole')
 df_MaritalStatus_one_hot = pd.get_dummies(df_combine['MaritalStatus'], prefix='MaritalStatus')
 df_OverTime_one_hot = pd.get_dummies(df_combine['OverTime'], prefix='OverTime',drop_first=True )
train_data['Transmission'].replace("Manual","MANUAL",inplace=True )<count_values>
df_nominal_catg = pd.concat([df_BusinessTravel_one_hot, df_Department_one_hot, df_EducationField_one_hot,
df_Gender_one_hot, df_JobRole_one_hot, df_MaritalStatus_one_hot,
df_OverTime_one_hot], axis=1)
 print(df_nominal_catg.shape)
 df_nominal_catg.head()
train_data['Transmission'].value_counts()<data_type_conversions>
final_catg_col = nominal_catg_col + ordinal_catg_col + ["Attrition"]
df_numeric = df_combine.drop(final_catg_col , axis=1)
 df_numeric.shape
train_data['Transmission'].fillna(value="Transmission_unk",inplace=True)
 test_data['Transmission'].fillna(value="Transmission_unk",inplace=True )<count_values>
numeric_col = list(df_numeric.columns)
 scaler = MinMaxScaler(feature_range=(0, 1))
df_numeric = scaler.fit_transform(df_numeric)
 df_numeric = pd.DataFrame(df_numeric, columns= numeric_col )
train_data['WheelTypeID'].value_counts()<drop_column>
df_pre_process = pd.concat([df_numeric, df_ordinal_catg.reset_index(drop=True),
df_nominal_catg.reset_index(drop=True)],axis=1,sort = False)
 print(df_pre_process.shape)
 df_pre_process.head()
train_data.drop('WheelTypeID',inplace=True,axis=1)
 test_data.drop('WheelTypeID',inplace=True,axis=1 )<count_values>
df_age = feat_to_binning("Age",5)
 print(df_age['Age_bin_round'].value_counts())
 df_age.head()
train_data['WheelType'].value_counts()<count_values>
X_train_pre['Age'] = df_age.loc[:1628,'Age_bin_round']
X_test_pre['Age'] = df_age.loc[1628:,'Age_bin_round']
test_data['WheelType'].value_counts()<data_type_conversions>
df_bin = pd.DataFrame()
df_con = pd.DataFrame()
train_data['WheelType'].fillna(value='WheelType_unk',inplace=True)
 test_data['WheelType'].fillna(value='WheelType_unk',inplace=True )<count_values>
bestfeatures = SelectKBest(score_func=chi2, k=21)
 fit = bestfeatures.fit(X,y )
train_data['WheelType'].value_counts()<count_values>
dfscores = pd.DataFrame(fit.scores_)
 dfcolumns = pd.DataFrame(X.columns)
 featureScores = pd.concat([dfcolumns,dfscores],axis=1)
 featureScores.columns = ['Specs','Score']
train_data['Nationality'].value_counts()<count_values>
featureScores
test_data['Nationality'].value_counts()<data_type_conversions>
print(featureScores.nlargest(21,'Score'))
train_data['Nationality'].fillna(value='Nationality_unk',inplace=True)
 test_data['Nationality'].fillna(value='Nationality_unk',inplace=True )<count_values>
bestfeatures_f = SelectKBest(score_func=f_classif, k=39)
 fit_f = bestfeatures.fit(X,y )
train_data['Size'].value_counts()<count_values>
dfscores_f = pd.DataFrame(fit_f.scores_)
 dfcolumns_f = pd.DataFrame(X.columns)
 featureScores_f = pd.concat([dfcolumns_f,dfscores_f],axis=1)
 featureScores_f.columns = ['Specs','Score']
test_data['Size'].value_counts()<data_type_conversions>
X_imp = X_train_pre
y_imp = y_train
model = xgboost.XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1.0, gamma=0.5, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.03, max_delta_step=0, max_depth=100,
min_child_weight=1, missing=None, monotone_constraints='() ',
n_estimators=500, n_jobs=0, num_parallel_tree=1,
objective='binary:logistic', random_state=0, reg_alpha=0,
reg_lambda=1, scale_pos_weight=1, subsample=0.8,
tree_method='exact', validate_parameters=1, verbosity=None)
 model.fit(X_imp,y_imp )
train_data['Size'].fillna(value='Size_unk',inplace=True)
 test_data['Size'].fillna(value="Size_unk",inplace=True )<count_values>
X_train = X_train_pre.drop(['Id','Behaviour','PerformanceRating', 'PercentSalaryHike',
'YearsAtCompany','TotalWorkingYears', 'EmployeeNumber',
'YearsSinceLastPromotion'], axis=1)
 X_test = X_test_pre.drop(['Id','Behaviour','PerformanceRating', 'PercentSalaryHike',
'YearsAtCompany','TotalWorkingYears', 'EmployeeNumber',
'YearsSinceLastPromotion'], axis=1)
 print(X_train.shape)
 print(X_test.shape )
train_data['TopThreeAmericanName'].value_counts()<count_values>
X_train_pre_cat = df_combine.iloc[:1628,:]
X_train_pre_cat = X_train_pre_cat.drop(['Attrition'], axis =1)
 X_test_pre_cat = df_combine.iloc[1628:,:]
X_test_pre_cat = X_test_pre_cat.drop(['Attrition'], axis =1)
 X_train_pre_cat['Age'] = df_age.loc[:1628,'Age_bin_round']
X_test_pre_cat['Age'] = df_age.loc[1628:,'Age_bin_round']
print(X_train_pre_cat.shape)
 print(X_test_pre_cat.shape )
test_data['TopThreeAmericanName'].value_counts()<data_type_conversions>
X_train_cat = X_train_pre_cat.drop(['Id','Behaviour','PerformanceRating', 'PercentSalaryHike',
'YearsAtCompany','TotalWorkingYears', 'EmployeeNumber',
'YearsSinceLastPromotion'], axis=1)
 X_test_cat = X_test_pre_cat.drop(['Id','Behaviour','PerformanceRating', 'PercentSalaryHike',
'YearsAtCompany','TotalWorkingYears', 'EmployeeNumber',
'YearsSinceLastPromotion'], axis=1)
 print(X_train_cat.shape)
 print(X_test_cat.shape)
train_data['TopThreeAmericanName'].fillna(value='Top_unk',inplace=True)
 test_data['TopThreeAmericanName'].fillna(value='Top_unk',inplace=True )<count_values>
all_catg_col = ['Age'] + ordinal_catg_col + nominal_catg_col
indices_cat = []
for col in all_catg_col:
if(col in list(X_train_cat.columns)) :
indices_cat.append(X_train_cat.columns.get_loc(col))
indices_cat.sort()
print(indices_cat)
 train_pool_cat = Pool(X_train_cat,
y_train,
indices_cat )
train_data['PRIMEUNIT'].value_counts()<count_values>
model_cat = CatBoostClassifier(random_state = 51,eval_metric = 'AUC')
 random_grid_cat = {'learning_rate': [0.05, 0.08, 0.1, 0.15, 0.2, 0.3],
'depth': [4, 6,10,15,20,30,40,50,60,70],
'l2_leaf_reg': [1, 3, 5, 7, 9]}
randomized_search_cat = model_cat.randomized_search(random_grid_cat, train_pool_cat, cv=5,plot=True)
test_data['PRIMEUNIT'].value_counts()<data_type_conversions>
randomized_search_cat
train_data['PRIMEUNIT'].fillna(value="Prime_unk",inplace=True)
 test_data['PRIMEUNIT'].fillna(value="Prime_unk",inplace=True )<categorify>

 model_train = CatBoostClassifier(iterations=170, learning_rate=0.1, random_state = 51,eval_metric = 'AUC',loss_function ='CrossEntropy')
 model_train.fit(train_pool_cat)
 acc_catboost = round(model_train.score(X_train_cat, y_train)* 100, 2 )
train_data['AUCGUART'].replace("AGREEN","GREEN",inplace=True)
 test_data['AUCGUART'].replace("ARED","RED",inplace=True )<data_type_conversions>
cv_params = model_train.get_params()
cv_data = cv(train_pool_cat,
cv_params,
fold_count=5,
plot=True)
 acc_cv_catboost = round(np.max(cv_data['test-AUC-mean'])* 100, 2)
train_data['AUCGUART'].fillna(value="AUC_unk",inplace=True)
 test_data['AUCGUART'].fillna(value="AUC_unk",inplace=True )<drop_column>
print('train accuracy: ' , acc_catboost)
 print("CV Accuracy: " ,acc_cv_catboost)
 cv_data.head()
train_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1)
 test_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1 )<drop_column>
y_pred=model_train.predict_proba(X_test_cat)
 pred=pd.DataFrame(y_pred[:,1])
 sub_df=pd.read_csv('.. /input/summeranalytics2020/Sample_submission.csv')
 datasets=pd.concat([sub_df['Id'],pred],axis=1)
 datasets.columns=['Id','Attrition']
datasets.to_csv('Catboost_submission_temp.csv',index=False )
train_data.drop('PurchDate',axis=1,inplace=True)
 test_data.drop("PurchDate",axis=1,inplace=True )<drop_column>
xbg_classifier = xgboost.XGBClassifier(scoring = 'roc_auc', random_state = 51)
 booster=['gbtree']
hyperparameter_grid = {
'n_estimators' : [100, 500, 900, 1100, 1500],
'min_child_weight': [1, 2, 3, 5, 7, 9],
'gamma': [0.5, 1, 1.5, 2, 5],
'subsample': [0.6, 0.8, 1.0],
'colsample_bytree': [0.6, 0.8, 1.0],
'max_depth': [5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120],
'learning_rate' : [0.005,0.01,0.03, 0.05, 0.15,0.3, 0.45, 0.55]
}
folds = 5
skf = StratifiedKFold(n_splits=folds, shuffle = True, random_state = 51)
 random_cv = RandomizedSearchCV(estimator=xbg_classifier,
param_distributions=hyperparameter_grid,
cv=skf.split(X_train,y_train), n_iter=50,
scoring = 'roc_auc',n_jobs = 4,
verbose = 5,
return_train_score = True )
train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object'<drop_column>
random_cv.fit(X_train, y_train )
not_categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object']<feature_engineering>
classifier_t = random_cv.best_estimator_
print(classifier_t)
 classifier_t.fit(X_train,y_train )
for i in not_categorical:
maximum=np.max(train_data[i])
 train_data[i]=train_data[i]/maximum
maximum_test=np.max(test_data[i])
 test_data[i]=test_data[i]/maximum_test<drop_column>
acc_xgb = round(classifier_t.score(X_train, y_train)* 100, 2)
 train_pred = model_selection.cross_val_predict(classifier_t,
X_train,
y_train,
cv=5,
n_jobs = -1)
 acc_cv = round(metrics.accuracy_score(y_train, train_pred)* 100, 2)
 print(acc_xgb)
 print(acc_cv )
categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes=='object']<filter>
y_pred=classifier_t.predict_proba(X_test)
 pred=pd.DataFrame(y_pred[:,1])
 sub_df=pd.read_csv('.. /input/summeranalytics2020/Sample_submission.csv')
 datasets=pd.concat([sub_df['Id'],pred],axis=1)
 datasets.columns=['Id','Attrition']
datasets.to_csv('XBGC_random_submission_temp.csv',index=False )