Kaggle competition: Flatiron School
<SOS> metric: mse Kaggle data source: flatiron-school<train_on_grid>
import numpy as np
import pandas as pd
import os
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
import statsmodels.api as sm
%%time
stack_gen = StackingCVRegressor(
 regressors=(
 lightgbm,
supv,
XGBoost,
lightgbm,
ridge,
gbr,
elasticnet,
lasso 
),
meta_regressor=XGBoost,
use_features_in_secondary=True 
)
 stack_gen_model = stack_gen.fit(np.array(X_test), np.array(y_test))
stack_predictions = stack_gen_model.predict(np.array(X_test))<predict_on_test>
train_df = pd.read_csv('.. /input/train.csv', index_col=0)
 test_df = pd.read_csv('.. /input/test.csv', index_col=0)
 train_df.head()
def blend(data):
return(
(0.10 * supv.predict(data)) + 
(0.20 * XGBoost.predict(data)) + 
(0.15 * lightgbm.predict(data)) + 
(0.15 * gbr.predict(data)) + 
(0.10 * ridge.predict(data)) + 
(0.10 * elasticnet.predict(data)) + 
(0.10 * lasso.predict(data)) + 
(0.10 * stack_gen_model.predict(np.array(data)))
 )<compute_test_metric>
train_df.isna().sum()
def calc(model): return int(np.sqrt(metrics.mean_squared_error(y_test, model)))
 print('XGBoost \tRMSE: \t{}$'.format(calc(xgboost_predictions)))
 print('SVR \t\tRMSE: \t{}$'.format(calc(supv_predictions)))
 print('lightGBM \tRMSE: \t{}$'.format(calc(lightgbm_predictions)))
 print('GBR \t\tRMSE: \t{}$'.format(calc(gbr_predictions)))
 print('Ridge \t \tRMSE: \t{}$'.format(calc(ridge_predictions)))
 print('Elasticnet \tRMSE: \t{}$'.format(calc(elasticnet_predictions)))
 print('Lasso \t \tRMSE: \t{}$'.format(calc(lasso_predictions)))
 print('Stack_gen \tRMSE: \t{}$'.format(calc(stack_predictions)))
 print('Blended \tRMSE: \t{}$'.format(int(np.sqrt(metrics.mean_squared_error(y_test, blend(X_test))))))<prepare_output>
cleaned_train_df = train_df.drop(['Critic_Score', 'Critic_Count', 'User_Score', 'User_Count', 'Developer', 'Rating'], axis=1 )
res = pd.DataFrame(
 blended_predictions,
index=import_data('test' ).reset_index() ['Id'] 
)
 res.rename(columns={0:'SalePrice','index':'Id'},inplace=True)
 res.reset_index(inplace=True)
 res.head()<save_to_csv>
cleaned_train_df.Year_of_Release.fillna(cleaned_train_df.Year_of_Release.median() , inplace=True )
res.to_csv('subm.csv',index = 0 )<save_to_csv>
cleaned_train_df.Genre.value_counts()
for kol in preds.columns:
res = preds[kol].to_frame()
res.set_index(import_data('test' ).reset_index() ['Id'])
 res.rename(columns={kol:'SalePrice','index':'Id'},inplace=True)
 res.to_csv('{}.csv'.format(kol))<import_modules>
cleaned_train_df.Publisher.value_counts()
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D<load_from_csv>
cleaned_train_df.dropna(subset=['Genre', 'Publisher'], inplace=True )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
cleaned_train_df.isna().sum()
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<prepare_x_and_y>
cleaned_train_df.Year_of_Release = cleaned_train_df.Year_of_Release.astype('int64' )
Y_train = train['label']<drop_column>
cleaned_train_df['JP_Sales_sqrt'] = np.sqrt(cleaned_train_df.JP_Sales)
 cleaned_train_df['NA_Sales_sqrt'] = np.sqrt(cleaned_train_df.NA_Sales )
X_train = train.drop(labels=['label'], axis =1)
 del train<feature_engineering>
cleaned_train_df.drop(['JP_Sales_sqrt', 'NA_Sales_sqrt'], axis=1, inplace=True )
X_train = X_train/255.0
test = test/255.0<categorify>
cleaned_train_df.JP_Sales.replace({0: 0.001}, inplace=True)
 cleaned_train_df.NA_Sales.replace({0: 0.001}, inplace=True )
Y_train = to_categorical(Y_train , num_classes = 10 )<split>
cleaned_train_df.JP_Sales = np.log(cleaned_train_df.JP_Sales)
 cleaned_train_df.NA_Sales = np.log(cleaned_train_df.NA_Sales )
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state = 42 )<choose_model_class>
cleaned_train_df.drop('Publisher', axis=1, inplace=True )
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'same', activation = 'relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'same', activation = 'relu'))
model.add(MaxPool2D(pool_size =(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'same', activation = 'relu'))
model.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'same', activation = 'relu'))
model.add(MaxPool2D(pool_size =(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = 'relu'))
model.add(Dropout(0.25))
model.add(Dense(10, activation = 'softmax'))<choose_model_class>
platform_counts = cleaned_train_df.Platform.value_counts()
platform_counts
model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'] )<define_variables>
uncommon_platforms = cleaned_train_df.Platform.isin(platform_counts.index[platform_counts<200])
 cleaned_train_df.loc[uncommon_platforms, 'Platform'] = 'Other'
epochs = 30
batch_size = 86<train_model>
platform_cats = list(cleaned_train_df.Platform.unique())
 print(cleaned_train_df.Platform.nunique())
 cleaned_train_df.Platform.value_counts()
datagen = ImageDataGenerator(rotation_range = 10,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1 
)
 datagen.fit(X_train)
<train_model>
cleaned_train_df = pd.get_dummies(cleaned_train_df )
history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size = batch_size), epochs = epochs, validation_data =(X_val, Y_val), steps_per_epoch=X_train.shape[0] // batch_size )<predict_on_test>
X = cleaned_train_df.drop('NA_Sales', axis=1)
 y = cleaned_train_df.NA_Sales
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results, name = 'Label' )<save_to_csv>
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )<load_from_csv>
X_train, X_test, y_train, y_test = train_test_split(X,y )
%matplotlib inline
train = pd.read_csv('.. /input/train.csv', header = 0, dtype={'Age': np.float64})
 test = pd.read_csv('.. /input/test.csv' , header = 0, dtype={'Age': np.float64})
 full_data = [train, test]
print(train.info())
 train.head()<groupby>
def stepwise_selection(X, y,
initial_list=[],
threshold_in=0.01,
threshold_out = 0.05,
verbose=True):

 included = list(initial_list)
 while True:
changed=False
excluded = list(set(X.columns)-set(included))
new_pval = pd.Series(index=excluded)
 for new_column in excluded:
model = sm.OLS(y, sm.add_constant(pd.DataFrame(X[included+[new_column]])) ).fit()
new_pval[new_column] = model.pvalues[new_column]
best_pval = new_pval.min()
if best_pval < threshold_in:
best_feature = new_pval.idxmin()
included.append(best_feature)
 changed=True
if verbose:
print('Add {:30} with p-value {:.6}'.format(best_feature, best_pval))
model = sm.OLS(y, sm.add_constant(pd.DataFrame(X[included])) ).fit()
pvalues = model.pvalues.iloc[1:]
worst_pval = pvalues.max()
if worst_pval > threshold_out:
changed=True
worst_feature = pvalues.argmax()
included.remove(worst_feature)
 if verbose:
print('Drop {:30} with p-value {:.6}'.format(worst_feature, worst_pval))
if not changed:
break
return included
print(train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean())
 print(train[['Pclass', 'Survived']].groupby(['Pclass'] ).mean() )<groupby>
final_features = stepwise_selection(X_train, y_train )
print(train[["Sex", "Survived"]].groupby(['Sex'], as_index=False ).mean() )<groupby>
predictors = sm.add_constant(X_train[final_features])
 final_model = sm.OLS(y_train,predictors ).fit()
final_model.summary()
print(train[['SibSp', 'Survived']].groupby(['SibSp'], as_index=False ).mean())
 print(train[['Parch', 'Survived']].groupby(['Parch'], as_index=False ).mean() )<groupby>
linreg = LinearRegression()
linreg.fit(X_train[final_features], y_train)
 y_hat_train = linreg.predict(X_train[final_features])
 y_hat_test = linreg.predict(X_test[final_features])
 train_mse = mean_squared_error(y_train, y_hat_train)
 test_mse = mean_squared_error(y_test, y_hat_test)
 print("Train MSE:", train_mse)
 print("Test MSE:", test_mse )
for dataset in full_data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
print(train[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean() )<feature_engineering>
cleaned_test_df = test_df.drop(['Critic_Score', 'Critic_Count', 'User_Score', 'User_Count', 'Developer', 'Rating', 'Publisher'], axis=1)
 cleaned_test_df.JP_Sales.replace({0: 0.001}, inplace=True)
 cleaned_test_df.JP_Sales = np.log(cleaned_test_df.JP_Sales )
for dataset in full_data:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
print(train[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean() )<groupby>
plts = platform_cats
plts.remove('Other' )
for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S')
 print(train[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean() )<feature_engineering>
cleaned_test_df.loc[~cleaned_test_df['Platform'].isin(plts), 'Platform'] = 'Other'
for dataset in full_data:
dataset['Fare'] = dataset['Fare'].fillna(train['Fare'].median())
 dataset['CategoricalFare'] = pd.qcut(dataset['Fare'], 4)
 print(type(train['CategoricalFare'][0]))
print(train[['CategoricalFare', 'Survived']].groupby(['CategoricalFare'], as_index=False ).mean() )<data_type_conversions>
cleaned_test_df = pd.get_dummies(cleaned_test_df )
for dataset in full_data:
age_avg 	 = dataset['Age'].mean()
age_std 	 = dataset['Age'].std()
age_null_count = dataset['Age'].isnull().sum()
age_null_random_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_null_count)
 dataset['Age'][np.isnan(dataset['Age'])] = age_null_random_list
dataset['Age'] = dataset['Age'].astype(int)
 dataset['CategoricalAge'] = pd.cut(dataset['Age'], 5)
 print(train[['CategoricalAge', 'Survived']].groupby(['CategoricalAge'], as_index=False ).mean() )<feature_engineering>
cleaned_test_df = cleaned_test_df[final_features]
def get_title(name):
	title_search = re.search('([A-Za-z]+)\.', name)
 	
	if title_search:
		return title_search.group(1)
 	return ""
for dataset in full_data:
dataset['Title'] = dataset['Name'].apply(get_title)
 print(pd.crosstab(train['Title'], train['Sex']))<feature_engineering>
test_data_notnull = cleaned_test_df[cleaned_test_df.Year_of_Release.notnull() ]
for dataset in full_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 print(train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean() )<categorify>
test_data_null = cleaned_test_df[cleaned_test_df.Year_of_Release.isna() ]
for dataset in full_data:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] 						 = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] 							 = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 dataset.loc[ dataset['Age'] <= 16, 'Age'] 					 = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4<categorify>
test_data_notnull['Prediction'] = linreg.predict(test_data_notnull )
train = pd.get_dummies(train,columns=['Sex','Title','Embarked','Fare','Age'])
 test = pd.get_dummies(test,columns=['Sex','Title','Embarked','Fare','Age'] )<drop_column>
predictions = test_data_notnull['Prediction']
drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp',\
'Parch', 'FamilySize']
train = train.drop(drop_elements, axis = 1)
 train = train.drop(['CategoricalAge', 'CategoricalFare'], axis = 1)
 test = test.drop(drop_elements, axis = 1)
 test = test.drop(['CategoricalAge', 'CategoricalFare'], axis = 1)
 print(train.columns)
 print(full_data[0].columns )<define_variables>
predictions = pd.DataFrame(predictions )
train = train.values
test = test.values<import_modules>
test_data_null['Prediction'] = cleaned_train_df.NA_Sales.median()
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.metrics import accuracy_score, log_loss,f1_score
from xgboost import XGBClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier,ExtraTreesClassifier,VotingClassifier
from sklearn.naive_bayes import GaussianNB,BernoulliNB
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression<choose_model_class>
null_predictions = test_data_null['Prediction']
classifiers = [
KNeighborsClassifier(n_neighbors=5,weights='distance',p=2,n_jobs=-1),
SVC(probability=True,random_state=666,tol=1e-6),
DecisionTreeClassifier(min_samples_split=10,random_state=666),
RandomForestClassifier(n_estimators=500,random_state=666,n_jobs=-1),
	AdaBoostClassifier(n_estimators=500,random_state=666),
GradientBoostingClassifier(n_estimators=500,random_state=666,min_samples_split=10),
GaussianNB() ,
BernoulliNB() ,
LinearDiscriminantAnalysis(tol=1e-6),
QuadraticDiscriminantAnalysis(tol=1e-6),
LogisticRegression(tol=1e-6,random_state=666,class_weight='balanced',n_jobs=-1),
XGBClassifier(n_estimators=500,n_jobs=-1,random_state=666)]
log_cols = ["Classifier", "Accuracy","F1"]
log 	 = pd.DataFrame(columns=log_cols)
 sss = StratifiedShuffleSplit(n_splits=10, test_size=0.1, random_state=0)
 X = train[0::, 1::]
y = train[0::, 0]
acc_dict = {}
f1_dict = {}
for train_index, test_index in sss.split(X, y):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
for clf in classifiers:
name = clf.__class__.__name__
clf.fit(X_train, y_train)
 train_predictions = clf.predict(X_test)
 acc = accuracy_score(y_test, train_predictions)
 f1 = f1_score(y_test, train_predictions)
 if name in acc_dict:
acc_dict[name] += acc
else:
acc_dict[name] = acc
if name in f1_dict:
f1_dict[name] += f1
else:
f1_dict[name] = f1
for clf in acc_dict:
acc_dict[clf] = acc_dict[clf] / 10.0
f1_dict[clf] = f1_dict[clf] / 10.0
log_entry = pd.DataFrame([[clf, acc_dict[clf],f1_dict[clf]]], columns=log_cols)
 log = log.append(log_entry )<choose_model_class>
null_predictions = pd.DataFrame(null_predictions )
clf1 = KNeighborsClassifier(n_neighbors=5,weights='distance',p=2,n_jobs=-1)
 clf2 = SVC(probability=True,random_state=666,tol=1e-6)
 clf3 = DecisionTreeClassifier(min_samples_split=10,random_state=666)
 clf4 = RandomForestClassifier(n_estimators=500,random_state=666,n_jobs=-1)
 clf5 = AdaBoostClassifier(n_estimators=500,random_state=666)
 clf6 = GradientBoostingClassifier(n_estimators=500,random_state=666,min_samples_split=10)
 clf7 = BernoulliNB()
clf8 = LinearDiscriminantAnalysis(tol=1e-6)
 clf9 = LogisticRegression(tol=1e-6,random_state=666,class_weight='balanced',n_jobs=-1)
 clf10 = XGBClassifier(n_estimators=500,n_jobs=-1,random_state=666)
 optimal = VotingClassifier(estimators=[('1',clf1),('4',clf4),('5',clf5),('7',clf7),\ 
('9',clf9),('10',clf10)],n_jobs=-1,voting='hard')
 optimal = optimal.fit(train[:,1:],train[:,0])
 result = optimal.predict(test)
<save_to_csv>
predictions = predictions.append(null_predictions )
final_file = pd.DataFrame({'PassengerId':full_data[1].PassengerId,'Survived':result.astype(int)})
 final_file.to_csv('Titanic best working Classifier.csv',index=False )<load_from_csv>
predictions.Prediction = np.exp(predictions.Prediction )
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv" )<features_selection>
predictions['Id'] = predictions.index
train.columns[train.isnull().any() ], test.columns[test.isnull().any() ]<sort_values>
predictions = predictions[['Id', 'Prediction']]
<sort_values><EOS> 
predictions.to_csv('submission.csv', index=False )