Kaggle competition: Mercedes-Benz Greener Manufacturing
lasso = LogisticRegression(labelCol='Survived',
maxIter=100,
elasticNetParam=1,
regParam=0.0003)
 model = lasso.fit(train_df)
 pred = model.transform(valid_df)
 evaluator.evaluate(pred )<train_model>
warnings.filterwarnings("ignore")
 pd.set_option('display.max_rows', 1000)
 rcParams['figure.figsize'] = 8, 5
SEED=2020
kf = KFold(n_splits = 5, random_state = 2)
 InteractiveShell.ast_node_interactivity = "all"
rf = RandomForestClassifier(labelCol='Survived',
numTrees=100, maxDepth=3)
 model = rf.fit(train_df)
 pred = model.transform(valid_df)
 evaluator.evaluate(pred )<train_model>
train= pd.read_csv('/kaggle/input/mercedes-benz-greener-manufacturing/train.csv.zip')
 train.sample(20 )
gb = GBTClassifier(labelCol='Survived', maxIter=75, maxDepth=3)
 model = gb.fit(train_df)
 pred = model.transform(valid_df)
 evaluator.evaluate(pred )<count_missing_values>
one_value_cols = [col for col in train.columns if train[col].nunique() <= 1]
print(f'There are {len(one_value_cols)} columns in train dataset with one unique value.' )
for col in df2.columns:
print(col.ljust(20), df2.filter(df2[col].isNull() ).count() )<drop_column>
def basic_details(df):
b = pd.DataFrame()
b['Missing value'] = df.isnull().sum()
b['N unique value'] = df.nunique()
b['dtype'] = df.dtypes
return b
basic_details(train )
df2 = df2.fillna({'Embarked': 'S', 'Fare':14.45})
 df2 = df2.withColumn('FamilySize', df2['Parch'] + df2['SibSp'] ).\
drop('Parch', 'SibSp' )<drop_column>
train.drop(one_value_cols, axis=1, inplace=True)
df2 = age_imputer(df2, 'Mr', 33.02)
 df2 = age_imputer(df2, 'Mrs', 35.98)
 df2 = age_imputer(df2, 'Miss', 21.86)
 df2 = age_imputer(df2, 'Master', 4.75)
 df2 = df2.drop('Cabin', 'Name', 'Ticket', 'Title')
 df2.show(4 )<count_missing_values>
variance_treshold = 0.9
low_var_cols = [col for col in train.columns if train[col].value_counts(dropna=False, normalize=True ).values[0] > variance_treshold]
train=train.drop(low_var_cols, axis=1 )
for col in df2.columns:
print(col.ljust(20), df2.filter(df2[col].isNull() ).count() )<categorify>
def feature_creation(df):
for i in ['X0', 'X1', 'X2', 'X3', 'X5', 'X6', 'X8']:
for j in ['X0', 'X1', 'X2', 'X3', 'X5', 'X6', 'X8']:
df[i + "_" + j] = df[i].astype('str')+ "_" + df[j].astype('str')
 return df
train = feature_creation(train )
pred_test = model_final.transform(df2)
 predictions = pred_test.select('PassengerId', 'prediction')
 predictions = predictions.\
withColumn('Survived', predictions['prediction'].\
cast('integer')).drop('prediction')
 predictions.show(5 )<save_to_csv>
lencoders = {}
for col in train.select_dtypes(include=['object'] ).columns:
lencoders[col] = LabelEncoder()
train[col] = lencoders[col].fit_transform(train[col])
predictions.coalesce(1 ).write.csv('submission_file.csv', header=True )<load_from_csv>
train = train.drop_duplicates()
spark.read.csv('submission_file.csv', header=True ).show(4 )<save_to_csv>
train = train[(train['y'] <= 136)].reset_index(drop=True )
predictions.toPandas().to_csv('submission.csv', index=False )<save_model>
train["y"]=(( train["y"]-train["y"].min())/(train["y"].max() -train["y"].min()))
model_final.write().save('titanic_classification.model' )<import_modules>
X = train.drop('y', axis=1)
 y = train.y
X['n0'] =(X == 0 ).sum(axis=1)
 scaler = preprocessing.MinMaxScaler()
X = pd.DataFrame(scaler.fit_transform(X), columns = X.columns)
 X.sample(10 )
plotly.offline.init_notebook_mode()
%matplotlib inline
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<load_from_csv>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=SEED)
 rf = RandomForestRegressor(max_depth=4, n_estimators=5, random_state=SEED)
 cv = KFold(n_splits=10, shuffle=True, random_state=SEED)
 scores = cross_val_score(rf, X, y, scoring='r2', cv=cv, n_jobs=-1)
 s_mean = mean(scores)
 print('Mean R2: %.3f' %(s_mean))
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<feature_engineering>
regressor = xgb.XGBRegressor()
selector = RFECV(regressor, step = 1, cv=cv, n_jobs=-1,verbose=1, scoring='r2')
 selector.fit(X, y)
 print('The optimal number of features is {}'.format(selector.n_features_))
features_rfecv = [f for f,s in zip(X, selector.support_)if s]
print('The selected features are:')
 print('{}'.format(features_rfecv))
FirstDate = train.groupby('Country_Region' ).min() ['Date'].unique() [0]
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 while train[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate)].shape[0] > 0:
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1)
 train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'ConfirmedCases'] = train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'Last Confirm']
train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Fatalities'] = train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Last Fatalities']
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1 )<feature_engineering>
rf = RandomForestRegressor(max_depth=3, n_estimators=10, random_state=SEED)
 X_rfe = X[features_rfecv]
rf.fit(X_rfe, y)
 scores = cross_val_score(rf, X_rfe, y, cv=cv, scoring = 'r2')
 print("RF based on selected dataset")
 print("FR CV Accuracy Score after selection:", scores.mean().round(3))
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
adjusted_X_pred = X_pred[feature_use].values.reshape(-1, 1)
 model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(adjusted_X_train,adjusted_y_train_confirmed)
 y_hat_confirmed = model.predict(adjusted_X_pred)
 model.fit(adjusted_X_train,adjusted_y_train_fatalities)
 y_hat_fatalities = model.predict(adjusted_X_pred)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_1 = df_val.copy()<compute_test_metric>
rf.fit(X_rfe, y)
 perm = PermutationImportance(rf, random_state=SEED ).fit(X_rfe, y)
 eli5.show_weights(perm, feature_names = X_rfe.columns.tolist() )
<compute_test_metric><EOS> 
def mae(y_true, y_pred):
return np.mean(abs(y_true - y_pred))
baseline_guess = np.median(y)
 print('The baseline guess based on Y-median value %0.2f' % baseline_guess)
 print("Baseline Performance based on Y-median value: MAE = %0.4f" % mae(y_test, baseline_guess))
print("Baseline Performance based on Random Forest RFECV-model: MAE = %0.4f" % mae(y_test, rf.predict(X_test[features_rfecv])) )