Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_test_metric>
warnings.filterwarnings('ignore', category = DeprecationWarning)
 warnings.filterwarnings('ignore', category = FutureWarning)
 sns.set(style='white', context='notebook', palette='deep')
 plt.style.use('bmh')
 train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv')
 IDtest = test["PassengerId"]
train.shape
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<compute_test_metric>
def IQR_outlier(df,n,features):
outlier_indices=[]
for col in features:
Q1=np.percentile(df[col],25)
 Q3=np.percentile(df[col],75)
 IQR=Q3-Q1
outlier_step=1.5*IQR
outlier_list_col=df[(df[col]<Q1-outlier_step)|(df[col]>Q3+outlier_step)].index
print('Total Number Outliers of',col,' : ',len(outlier_list_col))
print('Percentage of Outliers of',col,' : ',np.round(len(outlier_list_col)/len(df[col])*100),'%')
 outlier_indices.extend(outlier_list_col)
 outlier_indices=Counter(outlier_indices)
 multiple_outliers=list(k for k, v in outlier_indices.items() if v>n)
 return multiple_outliers
num_features=['Age','SibSp','Parch','Fare']
Outliers_to_drop=IQR_outlier(train,2,num_features)
 print('Total number of Outlier indices : ', len(Outliers_to_drop))
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))<load_from_csv>
train.loc[Outliers_to_drop,num_features]
%%time
print('Train')
 train = pd.read_csv(".. /input/train/train.csv")
 print(train.shape)
 print('Test')
 test = pd.read_csv(".. /input/test/test.csv")
 print(test.shape)
 print('Breeds')
 breeds = pd.read_csv(".. /input/breed_labels.csv")
 print(breeds.shape)
 print('Colors')
 colors = pd.read_csv(".. /input/color_labels.csv")
 print(colors.shape)
 print('States')
 states = pd.read_csv(".. /input/state_labels.csv")
 print(states.shape )<prepare_x_and_y>
dataset["Fare"].isnull().sum()
target = train['AdoptionSpeed']
train_id = train['PetID']
test_id = test['PetID']
train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 test.drop(['PetID'], axis=1, inplace=True )<feature_engineering>
dataset["Fare"]=dataset["Fare"].fillna(dataset["Fare"].median() )
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train.loc[:, 'doc_sent_mag'] = doc_sent_mag
train.loc[:, 'doc_sent_score'] = doc_sent_score
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test.loc[:, 'doc_sent_mag'] = doc_sent_mag
test.loc[:, 'doc_sent_score'] = doc_sent_score<normalization>
dataset["Fare"] = dataset["Fare"].map(lambda i: np.log(i)if i > 0 else 0 )
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1, 
)
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=120)
 svd.fit(X)
 print(svd.explained_variance_ratio_.sum())
 print(svd.explained_variance_ratio_)
 X = svd.transform(X)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(120)])
 train = pd.concat(( train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(120)])
 test = pd.concat(( test, X_test), axis=1 )<normalization>
dataset['Sex']=dataset['Sex'].map({'male':0,'female':1} )
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=3, max_features=10000,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=120)
 svd.fit(X)
 print(svd.explained_variance_ratio_.sum())
 print(svd.explained_variance_ratio_)
 X = svd.transform(X)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(120)])
 train = pd.concat(( train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(120)])
 test = pd.concat(( test, X_test), axis=1 )<prepare_x_and_y>
print('Number of Null entries: ',dataset['Embarked'].isnull().sum())
 print('Most common dock: ',dataset.Embarked.mode() [0] )
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train.loc[:, 'vertex_x'] = vertex_xs
train.loc[:, 'vertex_y'] = vertex_ys
train.loc[:, 'bounding_confidence'] = bounding_confidences
train.loc[:, 'bounding_importance'] = bounding_importance_fracs
train.loc[:, 'dominant_blue'] = dominant_blues
train.loc[:, 'dominant_green'] = dominant_greens
train.loc[:, 'dominant_red'] = dominant_reds
train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train.loc[:, 'dominant_score'] = dominant_scores
train.loc[:, 'label_description'] = label_descriptions
train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test.loc[:, 'vertex_x'] = vertex_xs
test.loc[:, 'vertex_y'] = vertex_ys
test.loc[:, 'bounding_confidence'] = bounding_confidences
test.loc[:, 'bounding_importance'] = bounding_importance_fracs
test.loc[:, 'dominant_blue'] = dominant_blues
test.loc[:, 'dominant_green'] = dominant_greens
test.loc[:, 'dominant_red'] = dominant_reds
test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test.loc[:, 'dominant_score'] = dominant_scores
test.loc[:, 'label_description'] = label_descriptions
test.loc[:, 'label_score'] = label_scores<drop_column>
display(dataset.Cabin.shape)
 print('Number of null values : ', dataset.Cabin.isnull().sum())
 print('Percentage of null values : ', round(dataset.Cabin.isnull().sum() /dataset.Cabin.shape[0]*100),'%' )
%%time
train.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)
 test.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True )<data_type_conversions>
dataset.isnull().sum()
numeric_cols = ['Age', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'doc_sent_mag', 'doc_sent_score', 'dominant_score', 'dominant_pixel_frac', 'dominant_red', 'dominant_green', 'dominant_blue', 'bounding_importance', 'bounding_confidence', 'vertex_x', 'vertex_y', 'label_score'] + ['svd_{}'.format(i)for i in range(120)]
cat_cols = list(set(train.columns)- set(numeric_cols))
train.loc[:, cat_cols] = train[cat_cols].astype('category')
 test.loc[:, cat_cols] = test[cat_cols].astype('category')
 print(train.shape)
 print(test.shape)
 train.head()<train_on_grid>
index_NaN_age = dataset["Age"][dataset.Age.isnull() ].index
for i in index_NaN_age :
age_med = dataset["Age"].median()
age_pred = dataset["Age"][(( dataset['SibSp'] == dataset['SibSp'][i])&(dataset['Parch'] == dataset['Parch'][i])&(dataset['Pclass'] == dataset['Pclass'][i])) ].median()
if not np.isnan(age_pred):
dataset['Age'].iloc[i] = age_pred
else :
dataset['Age'].iloc[i] = age_med
n_repeats = 2
n_splits = 5
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = RepeatedStratifiedKFold(n_splits=n_splits, random_state=42, n_repeats = n_repeats)
 fold_splits = kf.split(train, target)
 cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], n_splits*n_repeats))
all_coefficients = np.zeros(( n_splits*n_repeats, 4))
feature_importance_df = pd.DataFrame()
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/'+str(n_splits*n_repeats))
if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target[dev_index], target[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = train.columns.values
fold_importance_df['importance'] = importances
fold_importance_df['fold'] = i
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 i += 1
print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 pred_full_test = pred_full_test /(n_splits*n_repeats)
 results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 80,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.01,
'min_child_samples': 150,
'min_child_weight': 0.1,
'verbosity': -1,
'data_random_seed': 3,
'early_stop': 100,
'verbose_eval': 100,
'num_rounds': 10000}
def runLGB(train_X, train_y, test_X, test_y, test_X2, params):
print('Prep LGB')
 d_train = lgb.Dataset(train_X, label=train_y)
 d_valid = lgb.Dataset(test_X, label=test_y)
 watchlist = [d_train, d_valid]
print('Train LGB')
 num_rounds = params.pop('num_rounds')
 verbose_eval = params.pop('verbose_eval')
 early_stop = None
if params.get('early_stop'):
early_stop = params.pop('early_stop')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 print("Valid Counts = ", Counter(test_y))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance() , coefficients, qwk
results = run_cv_model(train, test, target, runLGB, params, rmse, 'lgb' )<compute_train_metric>
dataset.isnull().sum()
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions )<predict_on_test>
print('Mean : ', dataset[['Cabin','Survived']].groupby('Cabin' ).mean())
 print('Count : ', dataset[['Cabin','Survived']].groupby('Cabin' ).count() )
optR = OptimizedRounder()
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions )<create_dataframe>
dataset['Title']=pd.Series([i.split(',')[1].split('.')[0].strip() for i in dataset.Name] )
pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)) , columns=list(range(5)) )<compute_test_metric>
dataset["Title"] = dataset["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset["Title"] = dataset["Title"].replace(['Ms', 'Mlle'], 'Miss')
 dataset["Title"] = dataset["Title"].replace(['Mme'], 'Mrs' )
quadratic_weighted_kappa(target, train_predictions )<compute_test_metric>
Ticket=[]
for i in list(dataset['Ticket']):
if i.isdigit() :
Ticket.append('X')
 else:
Ticket.append(i.split(' ')[0])
 dataset['Ticket']=Ticket
rmse(target, [r[0] for r in results['train']] )<prepare_output>
Ticket = []
for i in list(dataset.Ticket):
if not i.isdigit() :
Ticket.append(i.replace(".","" ).replace("/","" ).strip())
 else:
Ticket.append(i)
 dataset['Ticket']=Ticket
submission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})
 submission.head()<save_to_csv>
dataset = pd.get_dummies(dataset, columns = ["Cabin"], prefix="Cab")
 dataset = pd.get_dummies(dataset, columns = ["Embarked"], prefix="Em")
 dataset = pd.get_dummies(dataset, columns = ["Fsize"], prefix="Fam")
 dataset = pd.get_dummies(dataset, columns = ["Pclass"], prefix="Pc")
 dataset = pd.get_dummies(dataset, columns = ["Title"], prefix="Title")
 dataset = pd.get_dummies(dataset, columns = ["Ticket"], prefix="Tick" )
submission.to_csv('submission.csv', index=False )<save_to_csv>
dataset.drop(labels = ['Name','PassengerId'], axis = 1, inplace = True )
submission.to_csv('submission.csv', index=False )<set_options>
Y_train=dataset[:train_len]['Survived']
X_train=data[:train_len]
test=data[train_len:]
warnings.filterwarnings("ignore")
 %matplotlib inline<define_variables>
LABELS = ["isFraud"]
all_files = glob.glob(".. /input/lgmodels/*.csv")
 all_files<load_from_csv>
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve
outs = [pd.read_csv(f, index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "m" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True )<feature_engineering>
kfold = StratifiedKFold(n_splits=10 )
rank = np.tril(concat_sub.iloc[:,1:].corr().values,-1)
 m =(rank>0 ).sum()
m_gmean, s = 0, 0
for n in range(min(rank.shape[0],m)) :
mx = np.unravel_index(rank.argmin() , rank.shape)
 w =(m-n)/(m+n)
 print(w)
 m_gmean += w*(np.log(concat_sub.iloc[:,mx[0]+1])+np.log(concat_sub.iloc[:,mx[1]+1])) /2
s += w
rank[mx] = 1
m_gmean = np.exp(m_gmean/s )<save_to_csv>
random_state = 42
classifiers = []
classifiers.append(SVC(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(RandomForestClassifier(random_state=random_state))
classifiers.append(KNeighborsClassifier())
 classifiers.append(LogisticRegression(random_state = random_state))
cv_results = []
for classifier in classifiers :
cv_results.append(cross_val_score(classifier, X_train, y = Y_train, scoring = "accuracy", cv = kfold, n_jobs=4))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,"Algorithm":["SVC","DecisionTree","RandomForest","KNeighbours","LogisticRegression"]})
 g = sns.barplot("CrossValMeans","Algorithm",data = cv_res, palette="Set3",orient = "h",**{'xerr':cv_std})
 g.set_xlabel("Mean Accuracy")
 g = g.set_title("Cross validation scores")
concat_sub['isFraud'] = m_gmean
concat_sub[['TransactionID','isFraud']].to_csv('stack_gmean.csv',
index=False, float_format='%.4g' )<set_options>
SVMC = SVC(probability=True)
 svc_param_grid = {'kernel': ['rbf'], 'gamma': [ 0.001, 0.01, 0.1, 1],'C': [1, 10, 50, 100,200,300, 1000]}
gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsSVMC.fit(X_train,Y_train)
 SVMC_best = gsSVMC.best_estimator_
gsSVMC.best_score_
warnings.filterwarnings('ignore' )<define_variables>
DTC = DecisionTreeClassifier()
dt_param_grid = {'max_features': ['auto', 'sqrt', 'log2'],'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
'min_samples_leaf':[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],'random_state':[42]}
gsDTC = GridSearchCV(DTC,param_grid = dt_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsDTC.fit(X_train,Y_train)
 DTC_best = gsDTC.best_estimator_
gsDTC.best_score_
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed )<define_variables>
RFC = RandomForestClassifier()
rf_param_grid = {"max_depth": [None],"max_features": [1, 3, 10],"min_samples_split": [2, 3, 10],"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],"n_estimators" :[100,300],"criterion": ["gini"]}
gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsRFC.fit(X_train,Y_train)
 RFC_best = gsRFC.best_estimator_
gsRFC.best_score_
SEED = 42
seed_everything(SEED)
 TARGET = 'isFraud'
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d' )<init_hyperparams>
LRC = LogisticRegression()
lr_param_grid = {'penalty':['l1', 'l2'],'C': np.logspace(0, 4, 10)}
gsLRC=GridSearchCV(LRC,param_grid=lr_param_grid,cv=kfold,scoring='accuracy', n_jobs= 4, verbose = 1)
 gsLRC.fit(X_train,Y_train)
 LRC_best = gsLRC.best_estimator_
gsLRC.best_score_
lgb_params = {
'objective':'binary',
'boosting_type':'gbdt',
'metric':'auc',
'n_jobs':-1,
'learning_rate':0.01,
'num_leaves': 2**8,
'max_depth':-1,
'tree_learner':'serial',
'colsample_bytree': 0.7,
'subsample_freq':1,
'subsample':0.7,
'n_estimators':20000,
'max_bin':255,
'verbose':-1,
'seed': SEED,
'early_stopping_rounds':100,
}<feature_engineering>
KNNC = KNeighborsClassifier()
knn_param_grid = {'n_neighbors':[3, 4, 5, 6, 7, 8],'leaf_size':[1, 2, 3, 5],
'weights':['uniform', 'distance'],'algorithm':['auto', 'ball_tree','kd_tree','brute']}
gsKNNC=GridSearchCV(KNNC,param_grid=knn_param_grid,cv=kfold,scoring='accuracy',n_jobs=4,verbose=1)
 gsKNNC.fit(X_train,Y_train)
 KNN_best=gsKNNC.best_estimator_
gsKNNC.best_score_
print('Load Data')
 train_df = pd.read_pickle('.. /input/ieee-data-minification/train_transaction.pkl')
 train_df['DT_M'] = train_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train_df['DT_M'] =(train_df['DT_M'].dt.year-2017)*12 + train_df['DT_M'].dt.month
test_df = train_df[train_df['DT_M']==train_df['DT_M'].max() ].reset_index(drop=True)
 train_df = train_df[train_df['DT_M']<(train_df['DT_M'].max())].reset_index(drop=True)
 print('Shape control:', train_df.shape, test_df.shape )<categorify>
votingC = VotingClassifier(estimators=[('svc', SVMC_best),('rfc', RFC_best),('lrc', LRC_best)], voting='soft', n_jobs=4)
 votingC = votingC.fit(X_train, Y_train )
for col in list(train_df):
if train_df[col].dtype=='O':
print(col)
 train_df[col] = train_df[col].fillna('unseen_before_label')
 test_df[col] = test_df[col].fillna('unseen_before_label')
 train_df[col] = train_df[col].astype(str)
 test_df[col] = test_df[col].astype(str)
 le = LabelEncoder()
le.fit(list(train_df[col])+list(test_df[col]))
train_df[col] = le.transform(train_df[col])
 test_df[col] = le.transform(test_df[col])
 train_df[col] = train_df[col].astype('category')
 test_df[col] = test_df[col].astype('category' )<define_variables>
rm_cols = [
'TransactionID','TransactionDT',
TARGET,
'DT_M'
]
rm_cols += ['V'+str(i)for i in range(1,340)]
features_columns = [col for col in list(train_df)if col not in rm_cols] 
<define_variables>
test_Survived = votingC.predict(test ).astype(int)
 submission = pd.DataFrame({
"PassengerId": IDtest,
"Survived": test_Survived
})
 submission.to_csv('Titanic_test_prediction_V9.csv', index=False )
<train_model><EOS> 
accuracy_score(Y_train,votingC.predict(X_train))