Kaggle competition: Mercedes-Benz Greener Manufacturing
all_data_rep_enc_1=all_data_rep_enc_1.drop(['Utilities'],axis=1 )<filter>
color = sns.color_palette()
%matplotlib inline
cat_feat=all_data_rep_enc_1.dtypes[all_data_rep_enc_1.dtypes== "object"].index
cat_feat<import_modules>
PATH = '.. /input/mercedes-benz-greener-manufacturing/'
!ls {PATH}
from sklearn import metrics
from sklearn.metrics import mean_squared_error
from sklearn import feature_selection
from catboost import CatBoostRegressor
from sklearn.model_selection import train_test_split
from sklearn import preprocessing<split>
df_raw = pd.read_csv(f'{PATH}train.csv.zip')
 test = pd.read_csv(f'{PATH}test.csv.zip' )
X_train_, X_valid, y_train_, y_valid = train_test_split(
 X_train, y_train, test_size=0.33, random_state=241 )<train_model>
display_all(df_raw.isnull().sum() /len(df_raw))
cb_model = CatBoostRegressor(iterations=3500,learning_rate=0.03,od_type='Iter',od_wait=1500,
depth=6,random_strength=1,
l2_leaf_reg=10,
sampling_frequency='PerTree', 
)
 zzz=cb_model.fit(X_train_, y_train_,
eval_set=(X_valid,y_valid),
cat_features=cat_feat,use_best_model=True,
verbose=True )<choose_model_class>
df, y, nas = proc_df(df_raw, 'y' )
model = cb_model
grid_par=[{'iterations':list(range(3138,3139)) }]
cv=KFold(n_splits=5,shuffle=True,random_state=241)
 gs = GridSearchCV(model,scoring='neg_mean_squared_error',cv=cv,param_grid=grid_par )<train_model>
m = RandomForestRegressor(n_jobs=-1)
 m.fit(df,y )
gs.fit(X_train, y_train,cat_features=cat_feat,use_best_model=True,verbose=True )<find_best_params>
m.score(df,y )
gs.cv_results_<train_model>
apply_cats(test, df_raw)
 sub = pd.DataFrame()
sub['ID'] = test.ID
model_out= CatBoostRegressor(iterations=3138,learning_rate=0.03,
depth=6,random_strength=1,
l2_leaf_reg=10,
sampling_frequency='PerTree')
 CAT=model_out.fit(X_train, y_train,
cat_features=cat_feat,
verbose=True)
 test = x_test.copy()
cat_pred=CAT.predict(test )<prepare_output>
test_df, _, _ = proc_df(test, na_dict=nas)
 df, _, _ = proc_df(df, na_dict=nas )
finalMd = np.expm1(cat_pred)
 finalMd 
<save_to_csv>
def split_vals(a,n): return a[:n].copy() , a[n:].copy()
n_valid = 800
n_trn = len(df)-n_valid
raw_train, raw_valid = split_vals(df_raw, n_trn)
 X_train, X_valid = split_vals(df, n_trn)
 y_train, y_valid = split_vals(y, n_trn)
 X_train.shape, y_train.shape, X_valid.shape
sample_submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 sample_submission.iloc[:,1] = finalMd
sample_submission.to_csv("sample_submission.csv", index=False )<import_modules>
m = RandomForestRegressor(n_jobs=-1, max_samples=250, oob_score=True, n_estimators=120)
 %time m.fit(X_train, y_train)
 def print_score(m):
res = [m.score(X_train, y_train), m.score(X_valid, y_valid)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res)
 print_score(m)
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV
from sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score, roc_curve
from sklearn.preprocessing import StandardScaler, OneHotEncoder, KBinsDiscretizer
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.neural_network import MLPClassifier
import xgboost as xgb<set_options>
sub['y'] = m.predict(test_df)
 sub.to_csv('rf_models_fastai300all.csv', index=False )
warnings.filterwarnings("ignore" )<set_options>
color = sns.color_palette()
%matplotlib inline
%matplotlib inline<load_from_csv>
PATH = '.. /input/mercedes-benz-greener-manufacturing/'
!ls {PATH}
data = pd.read_csv('/kaggle/input/titanic/train.csv')
<find_best_params>
df_raw = pd.read_csv(f'{PATH}train.csv.zip')
 test = pd.read_csv(f'{PATH}test.csv.zip' )
def test_thresh(lower = 0.1, upper = 0.95, jump = 0.01):
accs = {}
for i in np.arange(lower, upper, jump):
accs[i] = accuracy_score(test[1], predict(test[0], model, transformers, thresh = i, prep = False)['Survived'])
 best_thresh = np.round(sorted(accs.items() , key = lambda x: x[1], reverse = True)[0][0], 2)
 train_acc = accuracy_score(train[1], predict(train[0], model, transformers, thresh = best_thresh, prep = False)['Survived'])
 print(f'Train score: {train_acc}')
 print('Top 5 thresholds(test score):')
 print(sorted(accs.items() , key = lambda x:(1-abs(0.5-x[0])) *x[1], reverse = True)[:5])
 return best_thresh<feature_engineering>
display_all(df_raw.isnull().sum() /len(df_raw))
def process_data(data, transformers = None):
data = data.copy()
data.set_index('PassengerId', inplace = True)
 data['title'] = data.Name.str.split(',' ).str[1].str.strip().str.split().str[0]
low_freq_titles = data.title.value_counts() [lambda x: x < 10].index
data['title'] = data['title'].apply(lambda x: 'Misc' if x in low_freq_titles else x)
 age_by_title = data.groupby('title' ).Age.median()
data['Age'] = data.apply(lambda x: age_by_title[x.title] if pd.isna(x.Age)else x.Age, axis = 1)
 data.loc[pd.isna(data.Embarked), 'Embarked'] = 'C'
data['fam_size'] = data.Parch + data.SibSp
data = data.fillna(method = 'ffill')
 vars_for_bins = ['Age', 'Fare']
vars_for_dummies = ['Sex', 'Embarked']
if transformers is not None:
bins = transformers[0]
dummy = transformers[1]
binned_age = bins.transform(np.array(data[vars_for_bins]))
dummies = dummy.transform(data[vars_for_dummies])
 else:
bins = KBinsDiscretizer()
dummy = OneHotEncoder(sparse = False, drop = 'first')
 binned_age = bins.fit_transform(np.array(data[vars_for_bins]))
dummies = dummy.fit_transform(data[vars_for_dummies])
 dummies = pd.DataFrame(dummies, columns = dummy.get_feature_names())
 binned_age = pd.DataFrame(binned_age.toarray() , columns = ['Age_'+str(i)for i in bins.bin_edges_[0]][:-1] + ['Fare_'+str(i)for i in bins.bin_edges_[1]][:-1])
 data.drop(['Cabin', 'Ticket', 'Name', 'Fare', 'Parch', 'SibSp', 'Age', 'Pclass', 'Sex', 'title', 'Embarked'], inplace = True, axis = 1, errors = 'ignore')
 data = pd.concat([data.reset_index() , dummies, binned_age], axis = 1 ).set_index('PassengerId')
 if transformers is None:
return data, bins, dummy
else:
return data<train_on_grid>
df, y, nas = proc_df(df_raw, 'y' )
def fit_model(train_data, scale = True):
preped, bins, dummy = process_data(train_data)
 scaler = StandardScaler()
X = preped.drop('Survived', axis = 1)
 if scale:
X = pd.DataFrame(scaler.fit_transform(X), columns = X.columns, index = X.index)
 else:
scaler = None
y = preped['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
 print('Training Neural Net')
 best_nn = RandomizedSearchCV(MLPClassifier() ,
verbose = 0,
n_iter = 30,
param_distributions = {'hidden_layer_sizes':[(50,50,),(100,100,100,),(300,300,),(50,50,50,50,),(50,50,50,50,50,),(200,200,200,)],
'activation':['relu','tanh'],
'solver':['adam','lbfgs'],
'learning_rate':['constant','adaptive'],
'learning_rate_init': np.logspace(np.log10(0.00001), np.log(1), 20),
'max_iter':[50,100,200,500],
'random_state':[0]})
 best_nn.fit(X_train, y_train)
 best_nn = best_nn.best_estimator_
print('Training XGB')
 best_xgb = RandomizedSearchCV(xgb.XGBClassifier() , n_iter = 30,
param_distributions = {'n_estimators':np.arange(50,550,50),
'booster':['gbtree', 'gblinear','dart'],
'max_depth':np.arange(1,50),
'learning_rate': np.logspace(np.log10(0.00001), np.log(1), 20),
'reg_alpha':np.logspace(np.log10(0.00001), np.log(1), 20)})
 best_xgb.fit(X_train, y_train)
 best_xgb = best_xgb.best_estimator_
print('Training Logistic Regression')
 best_log = RandomizedSearchCV(LogisticRegression() ,
error_score=0.0,
n_iter = 30,
param_distributions = {'penalty':['l1', 'l2', 'elasticnet','none'],
'dual':[True,False],
'C': np.logspace(np.log10(0.00001), np.log(1), 20),
'fit_intercept':[True, False],
'random_state':[0],
'max_iter':[30,50,100,200,500,1000],
'solver':['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']})
 best_log.fit(X_train, y_train)
 best_log = best_log.best_estimator_
print('Training Random Forest')
 best_forest = RandomizedSearchCV(RandomForestClassifier() , n_iter = 30, param_distributions = {'bootstrap': [True, False],
'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, None],
'max_features': ['auto', 'sqrt'],
'min_samples_leaf': [1, 2, 4],
'min_samples_split': [2, 5, 10],
'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000]})
 best_forest.fit(X_train, y_train)
 best_forest = best_forest.best_estimator_
best_estimators = [('Neural Net', best_nn),('XGB', best_xgb),('Logistic Reg', best_log),('Random Forest', best_forest)]
ensemble_model = VotingClassifier(best_estimators, voting = 'hard')
 ensemble_model.fit(X_train, y_train)
 print(f'Train Acc Score: {accuracy_score(y_train, ensemble_model.predict(X_train)) }')
 print(f'Test Acc Score: {accuracy_score(y_test, ensemble_model.predict(X_test)) }')
 return ensemble_model,(bins, dummy, scaler), preped,(X_train, y_train),(X_test, y_test )<predict_on_test>
m = RandomForestRegressor(n_jobs=-1)
 m.fit(df,y )
def predict(test_df, model, transformers, thresh = 0.5, prep = True):
if prep:
temp_df = process_data(test_df, transformers)
 idx = temp_df.index
if transformers[2] is not None:
temp_df = pd.DataFrame(transformers[2].transform(temp_df), columns = temp_df.columns)
 else:
temp_df = pd.DataFrame(test_df, columns = preped.drop('Survived',axis = 1 ).columns ).copy()
idx = temp_df.index
if thresh != 0.5:
predictions =(model.predict_proba(temp_df)[:,1] > thresh)* 1
return pd.DataFrame(columns = ['PassengerId','Survived'], data = [*zip(idx, predictions)])
 else:
return pd.DataFrame(columns = ['PassengerId','Survived'], data = [*zip(idx, model.predict(temp_df)) ] )<train_model>
m.score(df,y )
model, transformers, preped, train, test = fit_model(data, scale = True )<load_from_csv>
apply_cats(test, df_raw)
 sub = pd.DataFrame()
sub['ID'] = test.ID
test_set = pd.read_csv('/kaggle/input/titanic/test.csv')
 predictions = predict(test_set, model, transformers)
 predictions.head()<find_best_params>
test_df, _, _ = proc_df(test, na_dict=nas)
 df, _, _ = proc_df(df, na_dict=nas )
model.estimators_<save_to_csv>
def split_vals(a,n): return a[:n].copy() , a[n:].copy()
n_valid = 800
n_trn = len(df)-n_valid
raw_train, raw_valid = split_vals(df_raw, n_trn)
 X_train, X_valid = split_vals(df, n_trn)
 y_train, y_valid = split_vals(y, n_trn)
 X_train.shape, y_train.shape, X_valid.shape
predictions.to_csv('submission.csv', index = False )<load_from_csv>
m = RandomForestRegressor(n_jobs=-1, max_samples=250, oob_score=True, n_estimators=120)
 %time m.fit(X_train, y_train)
 def print_score(m):
res = [m.score(X_train, y_train), m.score(X_valid, y_valid)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res)
 print_score(m)
test_labels = pd.read_csv('/kaggle/input/titanic-solutions-for-selfscoring/pub')
 accuracy_score(test_labels['survived'], predictions['Survived'] )<load_from_csv>
sub['y'] = m.predict(test_df)
 sub.to_csv('rf_models_fastai300all.csv', index=False )