Kaggle competition: Don't Get Kicked!
xgb = XGBClassifier(learning_rate = 0.1,
max_depth = 6,
n_estimators = 100,
subsample = 0.8)
<train_model>
train_data=pd.read_csv('/kaggle/input/DontGetKicked/training.csv' )
xgb_model = xgb.fit(X,y )<predict_on_test>
test_data=pd.read_csv('/kaggle/input/DontGetKicked/test.csv')
 test_data.head()
test_pred = xgb_model.predict(test )<create_dataframe>
train_data.isnull().sum()
submission = pd.DataFrame({"PassengerId" : test_id, "Survived" : test_pred} )<save_to_csv>
test_data.isnull().sum()
submission.to_csv("submission.csv",index=False )<set_options>
train_data['IsBadBuy'].value_counts()
%matplotlib inline
py.init_notebook_mode(connected=True)
 GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier)
 warnings.filterwarnings('ignore' )<load_from_csv>
train_data['Model'].value_counts()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 PassengerId = test['PassengerId']
train.head(50 )<define_variables>
train_data.drop('Model',inplace=True,axis=1)
 test_data.drop('Model',inplace=True,axis=1 )
def detect_outliers(df,n,features):
outlier_indices = []
for col in features:
Q1 = np.percentile(df[col],25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(train,2,["Age","SibSp","Parch","Fare"])
 train.loc[Outliers_to_drop]<drop_column>
train_data['Trim'].value_counts()
<count_values>
train_data.drop('Trim',inplace=True,axis=1)
 test_data.drop('Trim',inplace=True,axis=1 )
for dataset in full_data:
dataset.loc[ dataset['Name_length'] <= 23, 'Name_length'] 						 = 0
dataset.loc[(dataset['Name_length'] > 23)&(dataset['Name_length'] <= 28), 'Name_length'] = 1
dataset.loc[(dataset['Name_length'] > 28)&(dataset['Name_length'] <= 40), 'Name_length'] = 2
dataset.loc[ dataset['Name_length'] > 40, 'Name_length'] 							 = 3
train['Name_length'].value_counts()<data_type_conversions>
train_data['SubModel'].value_counts()
for dataset in full_data:
dataset['Sex'] = dataset['Sex'].map({'female': 0, 'male': 1} ).astype(int )<data_type_conversions>
train_data.drop('SubModel',inplace=True,axis=1)
 test_data.drop('SubModel',inplace=True,axis=1 )
for dataset in full_data:
age_avg = dataset['Age'].mean()
age_std = dataset['Age'].std()
age_null_count = dataset['Age'].isnull().sum()
age_null_random_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_null_count)
 dataset['Age'][np.isnan(dataset['Age'])] = age_null_random_list
dataset['Age'] = dataset['Age'].astype(int)
 dataset.loc[ dataset['Age'] <= 14, 'Age'] 						 = 0
dataset.loc[(dataset['Age'] > 14)&(dataset['Age'] <= 30), 'Age'] = 5
dataset.loc[(dataset['Age'] > 30)&(dataset['Age'] <= 40), 'Age'] = 1
dataset.loc[(dataset['Age'] > 40)&(dataset['Age'] <= 50), 'Age'] = 3
dataset.loc[(dataset['Age'] > 50)&(dataset['Age'] <= 60), 'Age'] = 2
dataset.loc[ dataset['Age'] > 60, 'Age'] 							 = 4
train['Age'].value_counts()<sort_values>
train_data['Color'].value_counts()
train[["Age", "Survived"]].groupby(['Age'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<data_type_conversions>
test_data['Color'].value_counts()
for dataset in full_data:
dataset.loc[ dataset['Fare'] <= 2.7, 'Fare'] 						 = 0
dataset.loc[ dataset['Fare'] > 2.7, 'Fare'] 							 = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 train['Fare'].value_counts()<sort_values>
train_data['Color'].fillna(value='Unknown',inplace=True)
 test_data['Color'].fillna(value='Unknown',inplace=True )
for dataset in full_data:
dataset['Has_Cabin'] = dataset["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 train[["Has_Cabin", "Survived"]].groupby(['Has_Cabin'], as_index=False ).sum().sort_values(by='Survived', ascending=False )<sort_values>
train_data['Transmission'].value_counts()
train[["Has_Cabin", "Survived"]].groupby(['Has_Cabin'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<categorify>
test_data['Transmission'].value_counts()
for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S')
 dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_pivot = pd.pivot_table(train, values= 'Survived',index=['Embarked'],columns='Pclass',aggfunc=np.mean, margins=True)
 def color_negative_red(val):
color = 'red' if val < 0.4 else 'black'
return 'color: %s' % color
train_pivot = train_pivot.style.applymap(color_negative_red)
 train_pivot<count_values>
train_data[train_data['Transmission']=='Manual']
dataset['Embarked'] = dataset['Embarked'].replace(['0', '2'], '0')
 train['Fare'].value_counts()<categorify>
train_data['Transmission'].replace('Manual','MANUAL',inplace=True )
for dataset in full_data:
dataset['Title'] = dataset['Title'].replace(['Mrs', 'Miss'], 'MM')
 dataset['Title'] = dataset['Title'].replace(['Dr', 'Major', 'Col'], 'DMC')
 dataset['Title'] = dataset['Title'].replace(['Don', 'Rev', 'Capt', 'Jonkheer'],'DRCJ')
 dataset['Title'] = dataset['Title'].replace(['Mme', 'Ms', 'Lady', 'Sir', 'Mlle', 'Countess'],'MMLSMC')
 title_mapping = {"MM": 1, "Master":2, "Mr": 5, "DMC": 4, "DRCJ": 3, "MMLSMC": 0}
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(3)
 g = sns.FacetGrid(train, col='Survived')
 g = g.map(sns.distplot, "Age" )<sort_values>
train_data['Transmission'].value_counts()
train[["Title", "Survived"]].groupby(['Title'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<categorify>
train_data['Transmission'].fillna(value='TranUnknown',inplace=True)
 test_data['Transmission'].fillna(value='TranUnknown',inplace=True )
deck = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7, "U": 8}
for dataset in full_data:
dataset['Cabin'] = dataset['Cabin'].fillna("U0")
 dataset['Deck'] = dataset['Cabin'].map(lambda x: re.compile("([a-zA-Z]+)" ).search(x ).group())
 dataset['Deck'] = dataset['Deck'].map(deck)
 dataset['Deck'] = dataset['Deck'].fillna(0)
 dataset['Deck'] = dataset['Deck'].astype(int)
 train['Deck'].value_counts()<sort_values>
train_data['WheelTypeID'].value_counts()
for dataset in full_data:
dataset.loc[ dataset['Deck'] <= 1, 'Deck'] = 1
dataset.loc[(dataset['Deck'] > 1)&(dataset['Deck'] <= 6), 'Deck'] = 3
dataset.loc[ dataset['Deck'] > 6, 'Deck'] = 0
train[["Deck", "Survived"]].groupby(['Deck'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
train_data.drop('WheelTypeID',inplace=True,axis=1)
 test_data.drop('WheelTypeID',inplace=True,axis=1 )
train[['Pclass', 'Sex', 'Age', 'Parch', 'Fare', 'Embarked', 'Has_Cabin', 'FamilySize', 'Title', 'Survived']].groupby(['Survived'], as_index=False ).mean().sort_values(by='Pclass', ascending=False )<feature_engineering>
train_data['WheelType'].value_counts()
<sort_values>
train_data['WheelType'].value_counts()
train[["FamilySize", "Survived"]].groupby(['FamilySize'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<feature_engineering>
train_data['Nationality'].fillna(value='NatUnknown',inplace=True)
 test_data['Nationality'].fillna(value='NatUnknown',inplace=True )
for dataset in full_data:
dataset['Gender_Embarked'] = 0
dataset.loc[(dataset['Sex']==0)&(dataset['Embarked']==0), 'Gender_Embarked'] = 0
dataset.loc[(dataset['Sex']==0)&(dataset['Embarked']==2), 'Gender_Embarked'] = 1
dataset.loc[(dataset['Sex']==0)&(dataset['Embarked']==1), 'Gender_Embarked'] = 2
dataset.loc[(dataset['Sex']==1)&(dataset['Embarked']==2), 'Gender_Embarked'] = 3
dataset.loc[(dataset['Sex']==1)&(dataset['Embarked']==0), 'Gender_Embarked'] = 4
dataset.loc[(dataset['Sex']==1)&(dataset['Embarked']==1), 'Gender_Embarked'] = 5
train[["Gender_Embarked", "Survived"]].groupby(['Gender_Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<drop_column>
train_data['Size'].fillna(value='SizeUnknown',inplace=True)
 test_data['Size'].fillna(value='SizeUnknown',inplace=True )
drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp', 'Boys', 'IsAlone', 'Embarked']
train = train.drop(drop_elements, axis = 1)
 test = test.drop(drop_elements, axis = 1 )<train_model>
train_data['TopThreeAmericanName'].fillna(value='T3AUnknown',inplace=True)
 test_data['TopThreeAmericanName'].fillna(value='T3AUnknown',inplace=True )
X_train = train.drop("Survived", axis=1)
 Y_train = train["Survived"]
X_train.shape, Y_train.shape, test.shape
X_train, x_test, Y_train, y_test = train_test_split(X_train, Y_train, test_size=0.3, random_state=101)
 X_test = test.copy() 
<compute_train_metric>
train_data['PRIMEUNIT'].fillna(value='PrimeUnknown',inplace=True)
 test_data['PRIMEUNIT'].fillna(value='PrimeUnknown',inplace=True )
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred1 = logreg.predict(x_test)
 acc_log = round(logreg.score(x_test, y_test)* 100, 2)
 acc_log<compute_test_metric>
train_data['AUCGUART'].value_counts()
print(classification_report(y_test, Y_pred1))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred1), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )<compute_train_metric>
test_data['AUCGUART'].value_counts()
svc=SVC()
svc.fit(X_train, Y_train)
 Y_pred2 = svc.predict(x_test)
 acc_svc = round(svc.score(x_test, y_test)* 100, 2)
 acc_svc<compute_test_metric>
train_data['AUCGUART'].fillna(value='AucguartUnknown',inplace=True)
 test_data['AUCGUART'].fillna(value='AucguartUnknown',inplace=True )
print(classification_report(y_test, Y_pred2))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred2), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )<compute_test_metric>
train_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice','MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice','MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice']
,inplace=True,axis=1)
 test_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice','MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice','MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice']
,inplace=True,axis=1 )
print(classification_report(y_test, knn_predictions))
cm = pd.DataFrame(confusion_matrix(y_test, knn_predictions), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )<predict_on_test>
train_data.drop('PurchDate',inplace=True,axis=1)
 test_data.drop('PurchDate',inplace=True,axis=1 )
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 Y_pred3 = gaussian.predict(x_test)
 acc_gaussian = round(gaussian.score(x_test, y_test)* 100, 2)
 acc_gaussian<compute_test_metric>
train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object'
print(classification_report(y_test, Y_pred3))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred3), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )<compute_train_metric>
non_cat=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object']
perceptron = Perceptron()
perceptron.fit(X_train, Y_train)
 Y_pred4 = perceptron.predict(x_test)
 acc_perceptron = round(perceptron.score(x_test, y_test)* 100, 2)
 acc_perceptron<compute_test_metric>
for i in non_cat:
maxi=np.max(train_data[i])
 train_data[i]/=maxi
maxi_test=np.max(test_data[i])
 test_data[i]/=maxi_test
print(classification_report(y_test, Y_pred4))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred4), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )<compute_train_metric>
cat=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes=='object']
cat
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 Y_pred5 = linear_svc.predict(x_test)
 acc_linear_svc = round(linear_svc.score(x_test, y_test)* 100, 2)
 acc_linear_svc<compute_test_metric>
train_data[cat[0]]
print(classification_report(y_test, Y_pred5))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred5), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )<compute_train_metric>
pd.get_dummies(train_data[cat[0]] )
sgd = SGDClassifier()
sgd.fit(X_train, Y_train)
 Y_pred6 = sgd.predict(x_test)
 acc_sgd = round(sgd.score(x_test, y_test)* 100, 2)
 acc_sgd<compute_test_metric>
for i in cat:
dummies=pd.get_dummies(train_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
train_data=pd.concat([train_data,dummies],axis=1)
 train_data.drop(i,inplace=True,axis=1)
 dummies=pd.get_dummies(test_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
test_data=pd.concat([test_data,dummies],axis=1)
 test_data.drop(i,inplace=True,axis=1 )
print(classification_report(y_test, Y_pred6))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred6), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )<compute_train_metric>
for i in train_data.drop('IsBadBuy',axis=1 ).columns:
if i not in test_data.columns:
test_data[i]=np.zeros(len(test_data))
for i in test_data.columns:
if i not in train_data.columns:
train_data[i]=np.zeros(len(train_data))
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 Y_pred7 = decision_tree.predict(x_test)
 acc_decision_tree = round(decision_tree.score(x_test, y_test)* 100, 2)
 acc_decision_tree<compute_test_metric>
test_data=test_data[train_data.drop("IsBadBuy",axis=1 ).columns]
print(classification_report(y_test, Y_pred7))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred7), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )<find_best_model_class>
X=train_data.drop(['RefId','IsBadBuy'],axis=1)
 y=train_data['IsBadBuy']
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 random_forest_predictions = random_forest.predict(x_test)
 acc_random_forest = round(random_forest.score(x_test, y_test)* 100, 2)
 test_Survived = pd.Series(random_forest_predictions, name="Survived")
 Submission2 = pd.concat([PassengerId,test_Survived],axis=1)
 acc_random_forest<compute_test_metric>
X_train,X_test,y_train,y_test=train_test_split(X,y,random_state=42 )
print(classification_report(y_test, random_forest_predictions))
cm = pd.DataFrame(confusion_matrix(y_test, random_forest_predictions), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )<choose_model_class>
KNN=KNeighborsClassifier(n_neighbors=9)
 KNN.fit(X_train,y_train )
kfold = StratifiedKFold(n_splits=10)
 random_state = 2
classifiers = []
classifiers.append(LogisticRegression(random_state = random_state))
classifiers.append(SVC(random_state=random_state))
classifiers.append(KNeighborsClassifier())
 classifiers.append(GaussianNB())
 classifiers.append(Perceptron(random_state=random_state))
classifiers.append(LinearSVC(random_state=random_state))
classifiers.append(SGDClassifier(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state = random_state))
classifiers.append(RandomForestClassifier(random_state = random_state))
cv_results = []
for classifier in classifiers :
cv_results.append(cross_val_score(classifier, X_train, y = Y_train, scoring = "accuracy", cv = kfold, n_jobs=4))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,"Algorithm":['Logistic Regression', 'KNN', 'Gaussian',
'Perceptron', 'linear SVC', 'SGD', 'Decision Tree','SVMC', 'Random Forest']})
 g = sns.barplot("CrossValMeans","Algorithm",data = cv_res, palette="Set3",orient = "h",**{'xerr':cv_std})
 g.set_xlabel("Mean Accuracy")
 g = g.set_title("Cross validation scores" )<train_on_grid>
KNN.score(X_test,y_test )
DTC = DecisionTreeClassifier()
adaDTC = AdaBoostClassifier(DTC, random_state=7)
 ada_param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
"base_estimator__splitter" : ["best", "random"],
"algorithm" : ["SAMME","SAMME.R"],
"n_estimators" :[1,2],
"learning_rate": [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3,1.5]}
gsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsadaDTC.fit(X_train,Y_train)
 adaDTC_best = gsadaDTC.best_estimator_
gsadaDTC.best_score_<train_on_grid>
td=test_data.drop('RefId',axis=1 )
ExtC = ExtraTreesClassifier()
ex_param_grid = {"max_depth": [None],
"max_features": [1, 3, 7],
"min_samples_split": [2, 3, 7],
"min_samples_leaf": [1, 3, 7],
"bootstrap": [False],
"n_estimators" :[300,600],
"criterion": ["gini"]}
gsExtC = GridSearchCV(ExtC,param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsExtC.fit(X_train,Y_train)
 ExtC_best = gsExtC.best_estimator_
gsExtC.best_score_<train_on_grid>
predict=KNN.predict(td )
GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
'n_estimators' : [100,200,300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [4, 8],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1] }
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsGBC.fit(X_train,Y_train)
 GBC_best = gsGBC.best_estimator_
gsGBC.best_score_<train_on_grid>
Subm=pd.DataFrame(data=predict,columns=['IsBadBuy'])
 Subm.head()
SVMC = SVC(probability=True)
 svc_param_grid = {'kernel': ['rbf'],
'gamma': [ 0.001, 0.01, 0.1, 1],
'C': [1,10,50,100,200,300, 1000]}
gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsSVMC.fit(X_train,Y_train)
 SVMC_best = gsSVMC.best_estimator_
gsSVMC.best_score_<train_on_grid>
Subm['RefId']=test_data['RefId']
Subm.set_index('RefId',inplace=True )
rf_param_grid = {"max_depth": [None],
"max_features": [1, 3, 7],
"min_samples_split": [2, 3, 7],
"min_samples_leaf": [1, 3, 7],
"bootstrap": [False],
"n_estimators" :[300,600],
"criterion": ["gini"]}
gsrandom_forest = GridSearchCV(random_forest,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsrandom_forest.fit(X_train,Y_train)
 random_forest_best = gsrandom_forest.best_estimator_
gsrandom_forest.best_score_<find_best_model_class>
Subm.to_csv('DGK_KNN' )