Kaggle competition: Instant Gratification
<SOS> metric: AUC Kaggle data source: instant-gratification<feature_engineering>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic' )
train['Fare'] = np.log1p(( train['Fare'] - train['Fare'].mean())/ train['Fare'].std())
 test['Fare'] = np.log1p(( test['Fare'] - test['Fare'].mean())/ test['Fare'].std() )<feature_engineering>
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = VarianceThreshold(threshold=2 ).fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}' )
train['c_mean'] = pd.Series(train.mean(axis=1), index=train.index)
 c_mean_max = train['c_mean'].max()
c_mean_min = train['c_mean'].min()
c_mean_scaled =(train.c_mean-c_mean_min)/ c_mean_max
train['c_mean_s'] = pd.Series(c_mean_scaled, index=train.index)
 del train['c_mean']
train['c_std'] = pd.Series(train.std(axis=1), index=train.index)
 c_std_max = train['c_std'].max()
c_std_min = train['c_std'].min()
c_std_scaled =(train.c_std-c_std_min)/ c_std_max
train['c_std_s'] = np.log1p(pd.Series(c_std_scaled, index=train.index))
del train['c_std']
test['c_mean'] = pd.Series(test.mean(axis=1), index=test.index)
 c_mean_max = test['c_mean'].max()
c_mean_min = test['c_mean'].min()
c_mean_scaled =(test.c_mean-c_mean_min)/ c_mean_max
test['c_mean_s'] = np.log1p(pd.Series(c_mean_scaled, index=test.index))
del test['c_mean']
test['c_std'] = pd.Series(test.std(axis=1), index=test.index)
 c_std_max = test['c_std'].max()
c_std_min = test['c_std'].min()
c_std_scaled =(test.c_std-c_std_min)/ c_std_max
test['c_std_s'] = pd.Series(c_std_scaled, index=test.index)
 del test['c_std']
print(train.shape, test.shape )<count_duplicates>
train.loc[oof > 0.99, 'target'] = 1
train.loc[oof < 0.01, 'target'] = 0
print(train.shape)
 train.drop_duplicates(inplace=True)
 print(train.shape )<data_type_conversions>
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 sel = VarianceThreshold(threshold=1.5 ).fit(data[cols])
 data2 = sel.transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof)
 print('QDA scores CV =',round(auc,5))
print(" <*> Debut")
 kDate = time.strftime('%d%m%y_%H%M%S',time.localtime())
 start = time.time()
y_train = train['Survived'].values.astype(np.float64)
 x_train = train.drop(['PassengerId', 'Survived'], axis=1 ).values.astype(np.float64)
 x_test = test.drop(['PassengerId'], axis=1 ).values.astype(np.float64)
 print('Shape train: {}
Shape test: {}
Shape Y: {}'.format(x_train.shape, x_test.shape, y_train.shape))
NSplit = 5
SliceTrain = 0.75
SliceTest = 0.25
models = []
NIter = 0
TScore = 0
print("Entrainement")
 rs = StratifiedShuffleSplit(n_splits=NSplit, random_state=99, test_size=SliceTest)
 for train_index, test_index in rs.split(x_train, y_train):
X_train = x_train[train_index]
Y_train = y_train[train_index]
X_valid = x_train[test_index]
Y_valid = y_train[test_index]
rfc_params = {}
rfc_params['n_estimators'] = 200
rfc_params['learning_rate'] = 0.015
rfc_params['max_depth'] = 250
rfc_params['max_features'] = "auto"
rfc_params['min_samples_split'] = 0.7
rfc_params['min_samples_leaf'] = 0.01
rfc_params['random_state'] = 0
rfc_params['verbose'] = 0
sum_score = 0
score = 0
clf = GradientBoostingClassifier(**rfc_params)
 clf.fit(X_train, Y_train)
 models.append(clf)
 score = clf.score(X_valid, Y_valid)
 print(" <*> Entrainement ",NIter," avec ", SliceTrain, " pour train et ",SliceTest," pour test - Score : ", score)
 TScore += score
NIter += 1
TScore /= NSplit
print(" <*> ---------------- Resultats CV ------------------ ")
 print(" <*> params : ",rfc_params)
 print(" <*> Score Moyenne training : ", TScore)
 print("Verification avec le train")
 score = 0
SCLOG = 0
NIter = 0
for clf in models:
PTrain = clf.predict(x_train)
 score = clf.score(x_train, y_train)
 SCLOG += score
NIter += 1
SCLOG /= NSplit
print(" <*> Score Moyenne Train : ", SCLOG)
 print("Predictions")
 NIter = 0
ctb_pred1 = []
for clf in models:
PTest = clf.predict(x_test)
 ctb_pred1.append(PTest)
 NIter += 1
PTest = [0] * len(ctb_pred1[0])
 for i in range(NSplit):
PTest += ctb_pred1[i]
PTest /= NSplit
print(pd.DataFrame(PTest ).head())
 end = time.time()
print(" <*> Duree : ",end - start)
 print(" <*> Fin" )<save_to_csv>
cat_dict = dict()
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 for i in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 cat_dict[i] = train3.shape[1]
print(" Mise a jour des colonnes submit")
 submit['Survived'] = np.clip(PTest, 0, 1 ).astype(int)
 localtime = time.localtime(time.time())
 WDate = str(localtime.tm_mday ).rjust(2, '0')+str(localtime.tm_mon ).rjust(2, '0')+str(localtime.tm_year)
 SUBFIC = SUBINT_DIR+"Titanic_GBR_"+str(kDate)+".csv"
print(" <*> Ecriture deb CSV/7z : ", SUBFIC)
 submit.to_csv(SUBFIC, index=False)
 print(" <*> Ecriture fin CSV/7z : ", SUBFIC )<set_options>
test['target'] = preds
oof_var = np.zeros(len(train))
preds_var = np.zeros(len(test))
for k in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 pca = PCA(n_components=cat_dict[k], random_state= 1234)
 pca.fit(train2p[cols])
 train3p = pca.transform(train2p[cols])
 train3 = pca.transform(train2[cols])
 test3 = pca.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_var[idx1[test_index3]] += clf.predict_proba(train3[test_index3,:])[:,1]
preds_var[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof_var)
 print('Pseudo Labeled QDA scores CV =',round(auc,5))
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus'] = False
def readCsv(file):
with open(file, encoding='utf8', mode='r')as f:
return pd.read_csv(f)
 def seeNorm(series):
sns.distplot(series)
 print('偏度系数', series.skew())
 print('峰度系数', series.kurt())
 def drawHeatMap(dataset, target, k=-1):
corrmat = dataset.corr()
cols = dataset.columns
if -1 != k:
cols = corrmat.nlargest(k, target)[target].index
cm = np.corrcoef(dataset[cols].values.T)
 sns.set(font_scale=1.25)
 hm = sns.heatmap(cm, cbar=True, annot = True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values,
xticklabels=cols.values, cmap='coolwarm')
 plt.show()
return cols
def drawPairplot(dataset, cols):
sns.set()
sns.pairplot(dataset[cols], size=2.5)
 plt.show()
def seeMissing(dataset):
total = dataset.isnull().sum()
percent =(total/dataset.isnull().count())*100
total = total.drop(total[total == 0].index)
 percent = percent.drop(percent[percent == 0].index)
 total = total.sort_values(ascending=False)
 percent = percent.sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Missing Ratio'])
 print(missing_data.head(20))
f, ax = plt.subplots(figsize=(15,12))
plt.xticks(rotation='90')
 if percent.count() != 0:
sns.barplot(x=percent.index, y =percent)
 plt.xlabel('Features', fontsize=15)
 plt.ylabel('Percent of missing values', fontsize=15)
 plt.title('Percent missing data by feature', fontsize=15)
 def drawScatter(dataset, colx, coly):

 fig, ax = plt.subplots()
ax.scatter(x=dataset[colx], y = dataset[coly])
 plt.xlabel(colx, fontsize=13)
 plt.ylabel(coly, fontsize=13)
 plt.show()
def drawDist(dataset, col, title=''):
print('skewness', dataset[col].skew())
 print('kurt', dataset[col].kurt())
 sns.distplot(dataset[col], fit = norm)
(mu, sigma)= norm.fit(dataset[col])
 print('mu = {:.2f} and sigma = {:.2f}'.format(mu, sigma))
plt.legend(['Normal dist.( $\mu=$ {:.2f} and $\sigma=$ {:.2f}'.format(mu, sigma)], loc='best')
 plt.ylabel('Frequency')
 plt.title('{} distribution'.format(title))
plt.show()
def drawQQ(dataset, col):
fig = plt.figure()
res = stats.probplot(dataset[col], plot=plt)
 plt.show()
def drawBox(dataset, xcol, ycol):
plt.figure(figsize=(18, 8))
sns.boxplot(x=dataset[xcol], y=dataset[ycol])
 def delCol(dataset, col):
dataset.drop(columns=col, axis=1, inplace=True)
 class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1)
 class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis = 1)
 class DataMining() :
def __init__(self):
dirname = '/kaggle/input/titanic'
self.train = readCsv('{}/train.csv'.format(dirname))
self.test = readCsv('{}/test.csv'.format(dirname))
self.idName = 'PassengerId'
self.targetName = 'Survived'
self.trainX = self.train.drop([self.idName, self.targetName], axis = 1)
 self.trainY = self.train[self.targetName]
self.testId = self.test[self.idName]
self.testX = self.test.drop(self.idName, axis = 1)
 def preprocessing(self):
self.fillMissing()
self.mergeDataset()
self.dataset = self.dataset.fillna(np.nan)
 self.dataset.drop(['Name'], axis=1, inplace=True)
 print(self.dataset.head())
 self.labelEncoding(['Sex', 'Ticket', 'Cabin', 'Embarked'])
 self.splitDataset()
def mergeDataset(self):
self.train_len = self.trainX.shape[0]
self.dataset = pd.concat([self.trainX, self.testX], axis=0 ).reset_index(drop=True)
 def splitDataset(self):
self.trainX = self.dataset[:self.train_len]
self.testX = self.dataset[self.train_len :]
self.trainY = self.trainY.astype(int)
 def fillMedian(self, col, kind='train'):
if kind != 'train':
self.testX[col] = self.testX[col].fillna(self.testX[col].median())
 else:
self.trainX[col] = self.trainX[col].fillna(self.trainX[col].median())
 def fillMode(self, col, kind='train'):
if kind != 'train':
self.testX[col] = self.testX[col].fillna(self.testX[col].mode() [0])
 else:
self.trainX[col] = self.trainX[col].fillna(self.trainX[col].mode() [0])
 def fillMissing(self):
self.fillMedian('Age')
 self.fillMode('Cabin')
 self.fillMode('Embarked')
 self.fillMode('Cabin', 'test')
 self.fillMode('Fare', 'test')
 self.fillMode('Age', 'test')
 def labelEncoding(self, col):
enc = LabelEncoder()
self.dataset[col] = self.dataset[col].apply(enc.fit_transform)
 def clearOutlier(self):
feats = self.train.dtypes[self.train.dtypes != 'object'].index
print(feats)
 for f in feats:
Q1 = self.train[f].quantile (.25)
 Q3 = self.train[f].quantile (.75)
 IQR = Q3 - Q1
threshold = 1.5 * IQR
self.train[(self.train[f] < Q1 - threshold)|(self.train[f] > Q3 + threshold)] = np.nan
def exploring(self):
self.preprocessing()
print('
train missing')
 print(self.trainX.isnull().sum())
 print('
test missing')
 print(self.testX.isnull().sum())
 print(self.dataset.head())
 tmp = pd.concat([self.trainX, self.trainY], axis=1 ).reset_index(drop=True)
 drawHeatMap(tmp, self.targetName, -1)
 def score(self, model, name=''):
n_folds = 9
kf = KFold(n_splits= n_folds, random_state=1, shuffle=True)
 score = cross_val_score(model, self.trainX.values, self.trainY, cv=kf)
 print("{}score: {:.2f}%".format(name, score.mean() *100.0))
def predict(self):
self.ensemble()
self.wrapAns()
def ensemble(self):
all_ans = []
for i in range(len(self.models)) :
self.models[i].fit(self.trainX, self.trainY)
 all_ans.append(self.models[i].predict(self.testX))
self.testY = all_ans[1]
def wrapAns(self):
submission = pd.DataFrame()
submission[self.idName] = self.testId
submission[self.targetName] = self.testY.astype('int')
 save_dir = '/kaggle/working'
submission.to_csv('{}/submission.csv'.format(save_dir), index=False)
 print('the result has been written into the file "submission.csv"')
 def modeling(self):
lr = LogisticRegression(random_state=1)
 rf = RandomForestClassifier(random_state=1, n_estimators=200, criterion = 'entropy', min_samples_leaf=1, min_samples_split=4)
 gboost = GradientBoostingClassifier(learning_rate=0.2,max_depth = 4,n_estimators =100)
 self.models = [
rf,
gboost
]
def evaluate(self):
for model in self.models:
self.score(model)
 dm = DataMining()
dm.preprocessing()
dm.modeling()
dm.predict() 
<set_options>
test['target'] = preds_var
oof_var2 = np.zeros(len(train))
preds_var2 = np.zeros(len(test))
for k in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_var2[idx1[test_index3]] += clf.predict_proba(train3[test_index3,:])[:,1]
preds_var2[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof_var2)
 print('Pseudo Labeled QDA scores CV =',round(auc,5))
%matplotlib inline<define_variables>
auc = roc_auc_score(train['target'],0.5*(oof_var+ oof_var2))
print('Pseudo Labeled QDA scores CV =',round(auc,5))
<load_pretrained><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = 0.5* preds_var + 0.5*preds_var2
sub.to_csv('submission.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()