Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<save_to_csv>
start_real = datetime.now()
np.random.seed(123 )
column_name = ['stack_lgb_' + str(i)for i in range(9)]
pd.DataFrame(oof, columns = column_name ).to_csv('oof_lgb.csv', index=False)
 pd.DataFrame(y_pred_test, columns = column_name ).to_csv('submit_lgb.csv', index=False )<import_modules>
def rmsle(Y, Y_pred):
assert Y.shape == Y_pred.shape
return np.sqrt(np.mean(np.square(Y_pred - Y)))
import gc
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, BatchNormalization, Dropout, ReLU, Input
from tensorflow.keras.utils import to_categorical
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import log_loss<load_from_csv>
%%time
train_df = pd.read_table('.. /input/train.tsv')
 test_df = pd.read_table('.. /input/test.tsv')
 print(train_df.shape, test_df.shape )
train = pd.read_csv(".. /input/otto-group-product-classification-challenge/train.csv")
 test = pd.read_csv(".. /input/otto-group-product-classification-challenge/test.csv" )<choose_model_class>
train_df = train_df.drop(train_df[(train_df.price < 3.0)].index)
 train_df.shape
def build_model(_input_shape, _num_classes):
model = Sequential()
model.add(Dense(512, input_dim=_input_shape, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(512, input_dim=_input_shape, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(512, input_dim=_input_shape, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(512, activation='relu'))
model.add(Dense(_num_classes, activation="softmax"))
return model<normalization>
def preprocessing(_train, _test):
drop_cols = ["id"]
target_col = "target"
feat_cols = [col for col in train.columns if col not in drop_cols + [target_col]]
data = pd.concat([_train, _test] ).reset_index()
data_scale = StandardScaler().fit_transform(data[feat_cols])
 data = pd.concat([data[drop_cols + [target_col]], pd.DataFrame(data_scale)], axis=1)
 _train = data[~data[target_col].isnull() ].reset_index(drop=True)
 _test = data[data[target_col].isnull() ].reset_index(drop=True)
 _target = _train[target_col]
_train.drop(columns=drop_cols + [target_col], inplace=True)
 _test.drop(columns=drop_cols + [target_col], inplace=True)
 _target = LabelEncoder().fit_transform(target)
 input_shapes = _train.shape[1]
num_classes = np.unique(_target ).size
del data
gc.collect()
return _train, _test, _target<categorify>
%%time
def wordCount(text):
try:
if text == 'No description yet':
return 0
else:
text = text.lower()
words = [w for w in text.split(" ")]
return len(words)
 except:
return 0
train_df['desc_len'] = train_df['item_description'].apply(lambda x: wordCount(x))
test_df['desc_len'] = test_df['item_description'].apply(lambda x: wordCount(x))
train_df['name_len'] = train_df['name'].apply(lambda x: wordCount(x))
test_df['name_len'] = test_df['name'].apply(lambda x: wordCount(x))
train_df.head()
drop_cols = ["id"]
target_col = "target"
feat_cols = [col for col in train.columns if col not in drop_cols + [target_col]]
data = pd.concat([train, test] ).reset_index()
data_scale = StandardScaler().fit_transform(data[feat_cols])
 data = pd.concat([data[drop_cols + [target_col]], pd.DataFrame(data_scale)], axis=1)
 train = data[~data[target_col].isnull() ].reset_index(drop=True)
 test = data[data[target_col].isnull() ].reset_index(drop=True)
 target = train[target_col]
train.drop(columns=drop_cols + [target_col], inplace=True)
 test.drop(columns=drop_cols + [target_col], inplace=True)
 target = LabelEncoder().fit_transform(target)
 input_shapes = train.shape[1]
num_classes = np.unique(target ).size
del data
gc.collect()<count_unique_values>
%%time
def split_cat(text):
try: return text.split("/")
 except: return("No Label", "No Label", "No Label")
 train_df['subcat_0'], train_df['subcat_1'], train_df['subcat_2'] = \
zip(*train_df['category_name'].apply(lambda x: split_cat(x)))
 test_df['subcat_0'], test_df['subcat_1'], test_df['subcat_2'] = \
zip(*test_df['category_name'].apply(lambda x: split_cat(x)) )
np.unique(target ).size<choose_model_class>
full_set = pd.concat([train_df,test_df])
 all_brands = set(full_set['brand_name'].values)
 train_df.brand_name.fillna(value="missing", inplace=True)
 test_df.brand_name.fillna(value="missing", inplace=True)
 premissing = len(train_df.loc[train_df['brand_name'] == 'missing'])
 def brandfinder(line):
brand = line[0]
name = line[1]
namesplit = name.split(' ')
 if brand == 'missing':
for x in namesplit:
if x in all_brands:
return name
if name in all_brands:
return name
return brand
train_df['brand_name'] = train_df[['brand_name','name']].apply(brandfinder, axis = 1)
 test_df['brand_name'] = test_df[['brand_name','name']].apply(brandfinder, axis = 1)
 found = premissing-len(train_df.loc[train_df['brand_name'] == 'missing'])
 print(found )
EPOCHS = 20
BATCH_SIZE = 512
NFOLDS = 5
RANDOM_STATE = 871972
folds = StratifiedKFold(n_splits=NFOLDS, shuffle=True,
random_state=RANDOM_STATE )<train_model>
train_df["target"] = np.log1p(train_df.price)
 train_df, dev_df = train_test_split(train_df, random_state=123, train_size=0.99)
 n_trains = train_df.shape[0]
n_devs = dev_df.shape[0]
n_tests = test_df.shape[0]
print("Training on", n_trains, "examples")
 print("Validating on", n_devs, "examples")
 print("Testing on", n_tests, "examples" )
y_pred_test = np.zeros(( len(test), 9))
oof = np.zeros(( len(train), 9))
score = 0
for fold_n,(train_index, valid_index)in enumerate(folds.split(train, y=target)) :
print('Fold', fold_n)
 X_train, X_valid = train.iloc[train_index], train.iloc[valid_index]
y_train, y_valid = target[train_index], target[valid_index]
model = build_model(input_shapes, num_classes)
 model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 history = model.fit(train, target, epochs=EPOCHS, batch_size=BATCH_SIZE)
 y_pred_valid = model.predict(X_valid)
 oof[valid_index] = y_pred_valid
score += log_loss(y_valid, y_pred_valid)
 y_pred_test += model.predict(test)/ NFOLDS
print('valid logloss average:', score / NFOLDS, log_loss(target, oof))<save_to_csv>
full_df = pd.concat([train_df, dev_df, test_df] )
sample_submit = pd.read_csv(".. /input/otto-group-product-classification-challenge/sampleSubmission.csv")
 submit = pd.concat([sample_submit[['id']], pd.DataFrame(y_pred_test)], axis = 1)
 submit.columns = sample_submit.columns
submit.to_csv('submit.csv', index=False )<save_model>
%%time
def fill_missing_values(df):
df.category_name.fillna(value="missing", inplace=True)
 df.brand_name.fillna(value="missing", inplace=True)
 df.item_description.fillna(value="missing", inplace=True)
 df.item_description.replace('No description yet',"missing", inplace=True)
 return df
print("Filling missing data...")
 full_df = fill_missing_values(full_df)
 print(full_df.category_name[1] )
np.save("keras_oof.npy", oof )<import_modules>
%%time
print("Processing categorical data...")
 le = LabelEncoder()
le.fit(full_df.category_name)
 full_df['category'] = le.transform(full_df.category_name)
 le.fit(full_df.brand_name)
 full_df.brand_name = le.transform(full_df.brand_name)
 le.fit(full_df.subcat_0)
 full_df.subcat_0 = le.transform(full_df.subcat_0)
 le.fit(full_df.subcat_1)
 full_df.subcat_1 = le.transform(full_df.subcat_1)
 le.fit(full_df.subcat_2)
 full_df.subcat_2 = le.transform(full_df.subcat_2)
 del le
print(os.listdir(".. /input")) 
<load_from_csv>
data = pd.read_csv('.. /input/train.csv')
 train_X = data.drop(["target","id"], axis=1 )<categorify>
%%time
print("Transforming text data to sequences...")
 raw_text = np.hstack([full_df.item_description.str.lower() , full_df.name.str.lower() , full_df.category_name.str.lower() ])
 print(" Fitting tokenizer...")
 tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text)
 print(" Transforming text to sequences...")
 full_df['seq_item_description'] = tok_raw.texts_to_sequences(full_df.item_description.str.lower())
 full_df['seq_name'] = tok_raw.texts_to_sequences(full_df.name.str.lower())
 del tok_raw
le = LabelEncoder()
le.fit(data["target"])
 train_y = le.transform(data["target"] )<split>
MAX_NAME_SEQ = 10
MAX_ITEM_DESC_SEQ = 75
MAX_CATEGORY_SEQ = 8
MAX_TEXT = np.max([
np.max(full_df.seq_name.max()),
np.max(full_df.seq_item_description.max()),
])+ 100
MAX_CATEGORY = np.max(full_df.category.max())+ 1
MAX_BRAND = np.max(full_df.brand_name.max())+ 1
MAX_CONDITION = np.max(full_df.item_condition_id.max())+ 1
MAX_DESC_LEN = np.max(full_df.desc_len.max())+ 1
MAX_NAME_LEN = np.max(full_df.name_len.max())+ 1
MAX_SUBCAT_0 = np.max(full_df.subcat_0.max())+ 1
MAX_SUBCAT_1 = np.max(full_df.subcat_1.max())+ 1
MAX_SUBCAT_2 = np.max(full_df.subcat_2.max())+ 1
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=0)
 for train_index, test_index in sss.split(train_X.values, train_y):
X_train = train_X.values[train_index]
X_val = train_X.values[test_index]
y_train = train_y[train_index]
y_val = train_y[test_index]<count_missing_values>
%%time
def get_rnn_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ),
'item_desc': pad_sequences(dataset.seq_item_description, maxlen=MAX_ITEM_DESC_SEQ),
'brand_name': np.array(dataset.brand_name),
'category': np.array(dataset.category),
'item_condition': np.array(dataset.item_condition_id),
'num_vars': np.array(dataset[["shipping"]]),
'desc_len': np.array(dataset[["desc_len"]]),
'name_len': np.array(dataset[["name_len"]]),
'subcat_0': np.array(dataset.subcat_0),
'subcat_1': np.array(dataset.subcat_1),
'subcat_2': np.array(dataset.subcat_2),
}
return X
train = full_df[:n_trains]
dev = full_df[n_trains:n_trains+n_devs]
test = full_df[n_trains+n_devs:]
X_train = get_rnn_data(train)
 Y_train = train.target.values.reshape(-1, 1)
 X_dev = get_rnn_data(dev)
 Y_dev = dev.target.values.reshape(-1, 1)
 X_test = get_rnn_data(test )
missing_val_count_by_column =(data.isnull().sum())
 print(missing_val_count_by_column.sum() )<count_values>
def root_mean_squared_logarithmic_error(y_true, y_pred):
first_log = K.log(K.clip(y_pred, K.epsilon() , None)+ 1.)
second_log = K.log(K.clip(y_true, K.epsilon() , None)+ 1.)
return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1)+0.0000001)
 def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)+0.0000001 )
data["target"].value_counts()<train_model>
np.random.seed(123)
 def new_rnn_model(lr=0.001, decay=0.0):
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 desc_len = Input(shape=[1], name="desc_len")
 name_len = Input(shape=[1], name="name_len")
 subcat_0 = Input(shape=[1], name="subcat_0")
 subcat_1 = Input(shape=[1], name="subcat_1")
 subcat_2 = Input(shape=[1], name="subcat_2")
 emb_name = Embedding(MAX_TEXT, 20 )(name)
 emb_item_desc = Embedding(MAX_TEXT, 60 )(item_desc)
 emb_brand_name = Embedding(MAX_BRAND, 10 )(brand_name)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 emb_desc_len = Embedding(MAX_DESC_LEN, 5 )(desc_len)
 emb_name_len = Embedding(MAX_NAME_LEN, 5 )(name_len)
 emb_subcat_0 = Embedding(MAX_SUBCAT_0, 10 )(subcat_0)
 emb_subcat_1 = Embedding(MAX_SUBCAT_1, 10 )(subcat_1)
 emb_subcat_2 = Embedding(MAX_SUBCAT_2, 10 )(subcat_2)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name)
 , Flatten()(emb_item_condition)
 , Flatten()(emb_desc_len)
 , Flatten()(emb_name_len)
 , Flatten()(emb_subcat_0)
 , Flatten()(emb_subcat_1)
 , Flatten()(emb_subcat_2)
 , rnn_layer1
, rnn_layer2
, num_vars
])
 main_l = Dropout(0.1 )(Dense(512,kernel_initializer='normal',activation='relu' )(main_l))
main_l = Dropout(0.1 )(Dense(256,kernel_initializer='normal',activation='relu' )(main_l))
main_l = Dropout(0.1 )(Dense(128,kernel_initializer='normal',activation='relu' )(main_l))
main_l = Dropout(0.1 )(Dense(64,kernel_initializer='normal',activation='relu' )(main_l))
output = Dense(1, activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name , item_condition,
num_vars, desc_len, name_len, subcat_0, subcat_1, subcat_2], output)
 optimizer = Adam(lr=lr, decay=decay)
 model.compile(loss = 'mse', optimizer = optimizer)
 return model
model = new_rnn_model()
model.summary()
del model
ros = RandomOverSampler()
X_ros, y_ros = ros.fit_sample(X_train, y_train)
 unique, counts = np.unique(y_ros, return_counts=True)
 print(np.asarray(( unique, counts)).T )<normalization>
%%time
BATCH_SIZE = 512 * 3
epochs = 2
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
steps = int(len(X_train['name'])/ BATCH_SIZE)* epochs
lr_init, lr_fin = 0.005, 0.001
lr_decay = exp_decay(lr_init, lr_fin, steps)
 rnn_model = new_rnn_model(lr=lr_init, decay=lr_decay)
 rnn_model.fit(
 X_train, Y_train, epochs=epochs, batch_size=BATCH_SIZE,
validation_data=(X_dev, Y_dev), verbose=1,
)
scaler = StandardScaler()
scaler.fit(X_train)
 X_train_scaled = scaler.transform(X_train)
 X_val_scaled = scaler.transform(X_val )<load_from_csv>
%%time
print("Evaluating the model on validation data...")
 Y_dev_preds_rnn = rnn_model.predict(X_dev, batch_size=BATCH_SIZE)
 print(" RMSLE error:", rmsle(Y_dev, Y_dev_preds_rnn))
test_data = pd.read_csv('.. /input/test.csv')
 test_X = test_data.drop(["id"], axis=1)
 scaler_all = StandardScaler()
train_X_scaled = scaler_all.fit_transform(train_X)
 test_X_scaled = scaler.transform(test_X )<normalization>
rnn_preds = rnn_model.predict(X_test, batch_size=BATCH_SIZE, verbose=1)
 rnn_preds = np.expm1(rnn_preds )
pca = decomposition.PCA(n_components=20)
 pca.fit(X_train_scaled)
 X_train_pca = pca.transform(X_train_scaled)
 print(pca.explained_variance_ratio_)
 print(pca.explained_variance_ )<compute_train_metric>
full_df = pd.concat([train_df, dev_df, test_df] )
xgb = XGBClassifier()
xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )<find_best_params>
%%time
print("Handling missing values...")
 full_df['category_name'] = full_df['category_name'].fillna('missing' ).astype(str)
 full_df['subcat_0'] = full_df['subcat_0'].astype(str)
 full_df['subcat_1'] = full_df['subcat_1'].astype(str)
 full_df['subcat_2'] = full_df['subcat_2'].astype(str)
 full_df['brand_name'] = full_df['brand_name'].fillna('missing' ).astype(str)
 full_df['shipping'] = full_df['shipping'].astype(str)
 full_df['item_condition_id'] = full_df['item_condition_id'].astype(str)
 full_df['desc_len'] = full_df['desc_len'].astype(str)
 full_df['name_len'] = full_df['name_len'].astype(str)
 full_df['item_description'] = full_df['item_description'].fillna('No description yet' ).astype(str )
xgb.get_params<train_on_grid>
%%time
print("Vectorizing data...")
 default_preprocessor = CountVectorizer().build_preprocessor()
def build_preprocessor(field):
field_idx = list(full_df.columns ).index(field)
 return lambda x: default_preprocessor(x[field_idx])
 vectorizer = FeatureUnion([ 
('name', CountVectorizer(
 ngram_range=(1, 2),
max_features=50000,
preprocessor=build_preprocessor('name'))), 
('subcat_0', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('subcat_0'))), 
('subcat_1', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('subcat_1'))), 
('subcat_2', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('subcat_2'))), 
('brand_name', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('brand_name'))), 
('shipping', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('shipping'))), 
('item_condition_id', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('item_condition_id'))), 
('desc_len', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('desc_len'))), 
('name_len', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('name_len'))), 
('item_description', TfidfVectorizer(
 ngram_range=(1, 3),
max_features=100000,
preprocessor=build_preprocessor('item_description'))),
])
 X = vectorizer.fit_transform(full_df.values)
 X_train = X[:n_trains]
Y_train = train_df.target.values.reshape(-1, 1)
 X_dev = X[n_trains:n_trains+n_devs]
Y_dev = dev_df.target.values.reshape(-1, 1)
 X_test = X[n_trains+n_devs:]
print(X.shape, X_train.shape, X_dev.shape, X_test.shape )
<train_model>
%%time
print("Fitting Ridge model on training examples...")
 ridge_model = Ridge(
 solver='auto', fit_intercept=True, alpha=1.0,
max_iter=100, normalize=False, tol=0.05, random_state = 1, 
)
 ridge_modelCV = RidgeCV(
 fit_intercept=True, alphas=[5.0],
normalize=False, cv = 2, scoring='neg_mean_squared_error', 
)
 ridge_model.fit(X_train, Y_train)
 ridge_modelCV.fit(X_train, Y_train )
scores = []
n_estimators = [100,200,400,450,500,525,550,600,700]
for nes in n_estimators:
xgb = XGBClassifier(learning_rate =0.1, n_estimators=nes, max_depth=7, min_child_weight=3, subsample=0.8,
colsample_bytree=0.8, nthread=4, seed=42, objective='multi:softprob')
 xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )<find_best_params>
Y_dev_preds_ridge = ridge_model.predict(X_dev)
 Y_dev_preds_ridge = Y_dev_preds_ridge.reshape(-1, 1)
 print("RMSL error on dev set:", rmsle(Y_dev, Y_dev_preds_ridge))
scores_md = []
max_depths = [1,3,5,6,7,8,10]
for md in max_depths:
xgb = XGBClassifier(learning_rate =0.1, n_estimators=n_estimators[np.argmin(scores)],
max_depth=md, min_child_weight=3, subsample=0.8,
colsample_bytree=0.8, nthread=4, seed=42, objective='multi:softprob')
 xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores_md.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )<predict_on_test>
Y_dev_preds_ridgeCV = ridge_modelCV.predict(X_dev)
 Y_dev_preds_ridgeCV = Y_dev_preds_ridgeCV.reshape(-1, 1)
 print("CV RMSL error on dev set:", rmsle(Y_dev, Y_dev_preds_ridgeCV))
scores_mcw = []
min_child_weights = [1,2,3,4,5]
for mcw in min_child_weights:
xgb = XGBClassifier(learning_rate =0.1, n_estimators=n_estimators[np.argmin(scores)],
max_depth=max_depths[np.argmin(scores_md)],
min_child_weight=mcw, subsample=0.8,
colsample_bytree=0.8, nthread=4, seed=42, objective='multi:softprob')
 xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores_mcw.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )<train_model>
%%time
ridge_preds = ridge_model.predict(X_test)
 ridge_preds = np.expm1(ridge_preds)
 ridgeCV_preds = ridge_modelCV.predict(X_test)
 ridgeCV_preds = np.expm1(ridgeCV_preds )
scores_ss = []
subsamples = [0.5,0.6,0.7,0.8,0.9,1]
for ss in subsamples:
xgb = XGBClassifier(learning_rate =0.1, n_estimators=n_estimators[np.argmin(scores)],
max_depth=max_depths[np.argmin(scores_md)],
min_child_weight=min_child_weights[np.argmin(scores_mcw)], subsample=ss,
colsample_bytree=0.8, nthread=4, seed=42, objective='multi:softprob')
 xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores_ss.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )<choose_model_class>
%%time
def aggregate_predicts3(Y1, Y2, Y3, ratio1, ratio2):
assert Y1.shape == Y2.shape
return Y1 * ratio1 + Y2 * ratio2 + Y3 *(1.0 - ratio1-ratio2)
scores_cb = []
colsample_bytrees = [0.5,0.6,0.7,0.8,0.9,1]
for cb in colsample_bytrees:
xgb = XGBClassifier(learning_rate =0.1, n_estimators=n_estimators[np.argmin(scores)],
max_depth=max_depths[np.argmin(scores_md)],
min_child_weight=min_child_weights[np.argmin(scores_mcw)],
subsample=subsamples[np.argmin(scores_ss)],
colsample_bytree=cb, nthread=4, seed=42, objective='multi:softprob')
 xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores_cb.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )<choose_model_class>
%%time
best1 = 0
best2 = 0
lowest = 0.99
for i in range(100):
for j in range(100):
r = i*0.01
r2 = j*0.01
if r+r2 < 1.0:
Y_dev_preds = aggregate_predicts3(Y_dev_preds_rnn, Y_dev_preds_ridgeCV, Y_dev_preds_ridge, r, r2)
 fpred = rmsle(Y_dev, Y_dev_preds)
 if fpred < lowest:
best1 = r
best2 = r2
lowest = fpred
Y_dev_preds = aggregate_predicts3(Y_dev_preds_rnn, Y_dev_preds_ridgeCV, Y_dev_preds_ridge, best1, best2 )
scores_eta = []
etas = [0.001,0.01,0.1,0.2,0.3,0.5,1]
for eta in etas:
xgb = XGBClassifier(learning_rate =eta, n_estimators=n_estimators[np.argmin(scores)],
max_depth=max_depths[np.argmin(scores_md)],
min_child_weight=min_child_weights[np.argmin(scores_mcw)],
subsample=subsamples[np.argmin(scores_ss)],
colsample_bytree=colsample_bytrees[np.argmin(scores_cb)],
nthread=4, seed=42, objective='multi:softprob')
 xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores_eta.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )<choose_model_class>
print(best1)
 print(best2)
 print("(Best)RMSL error for RNN + Ridge + RidgeCV on dev set:", rmsle(Y_dev, Y_dev_preds))
xgb = XGBClassifier(learning_rate =eta, n_estimators=n_estimators[np.argmin(scores)],
max_depth=max_depths[np.argmin(scores_md)],
min_child_weight=min_child_weights[np.argmin(scores_mcw)],
subsample=subsamples[np.argmin(scores_ss)],
colsample_bytree=colsample_bytrees[np.argmin(scores_cb)],
nthread=4, seed=42, objective='multi:softprob')
 calibrated_xgb = CalibratedClassifierCV(xgb, cv=5, method='isotonic')
 calibrated_xgb.fit(X_train_scaled, y_train)
 preds = calibrated_xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores_eta.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )<save_to_csv>
preds = aggregate_predicts3(rnn_preds, ridgeCV_preds, ridge_preds, best1, best2)
 submission = pd.DataFrame({
"test_id": test_df.test_id,
"price": preds.reshape(-1),
})
 submission.to_csv("./rnn_ridge_submission_best.csv", index=False )
xgb = XGBClassifier(learning_rate =0.1, n_estimators=525, max_depth=8, min_child_weight=3, subsample=0.7,
colsample_bytree=0.7, nthread=4, seed=42, objective='multi:softprob')
 my_model = CalibratedClassifierCV(xgb, cv=5, method='isotonic')
 my_model.fit(train_X_scaled,train_y)
 test_preds = my_model.predict_proba(test_X_scaled)
 output = pd.DataFrame(test_preds,columns=["Class_"+str(i)for i in range(1,10)])
 output.insert(loc=0, column='id', value=test_data.id)
 output.to_csv('submission.csv', index=False )<split>
preds = aggregate_predicts3(rnn_preds, ridgeCV_preds, ridge_preds, 0.334, 0.333)
 submission = pd.DataFrame({
"test_id": test_df.test_id,
"price": preds.reshape(-1),
})
 submission.to_csv("./rnn_ridge_submission_mean.csv", index=False )
train = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/train.csv')
 X = train.iloc[:, 1:94]
y = train[['target']]
print(train.groupby('target')['id'].count())
 y = pd.get_dummies(y, columns=['target'])
 X_train, X_dev_test, y_train, y_dev_test = train_test_split(X, y, test_size=0.2)
 X_dev, X_test, y_dev, y_test = train_test_split(X_dev_test, y_dev_test, test_size=0.5)
 print('X_train shape: %s' % str(X_train.shape))
print('y_train shape: %s' % str(y_train.shape))
print('X_dev shape: %s' % str(X_dev.shape))
print('y_dev shape: %s' % str(y_dev.shape))
print('X_test shape: %s' % str(X_test.shape))
print('y_test shape: %s' % str(y_test.shape)) 
<train_model>
preds = aggregate_predicts3(rnn_preds, ridgeCV_preds, ridge_preds, best1-0.1, best2+0.1)
 submission = pd.DataFrame({
"test_id": test_df.test_id,
"price": preds.reshape(-1),
})
 submission.to_csv("./rnn_ridge_submission_var_1.csv", index=False )
model = tf.keras.models.Sequential(
 [
tf.keras.layers.Dense(128, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.001)) ,
tf.keras.layers.Dense(128, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.001)) ,
tf.keras.layers.Dense(9, activation='softmax'),
] 
)
 model.compile(loss='categorical_crossentropy', optimizer=tf.keras.optimizers.Adam(1e-3), metrics=['accuracy'])
 history = model.fit(X_train, y_train, epochs=20, validation_data=(X_dev, y_dev), batch_size=64)
 loss = history.history['loss']
accuracy = history.history['accuracy']
val_loss = history.history['val_loss']
val_accuracy = history.history['val_accuracy']
iteration = range(1, len(history.history['loss'])+1)
 loss_df = pd.DataFrame({'loss': loss, 'val_loss': val_loss}, index=iteration)
 loss_df.plot.line()
acc_df = pd.DataFrame({'accuracy': accuracy, 'val_accuracy': val_accuracy}, index=iteration)
 acc_df.plot.line() 
<compute_test_metric>
preds = aggregate_predicts3(rnn_preds, ridgeCV_preds, ridge_preds, best1+0.1, best2-0.1)
 submission = pd.DataFrame({
"test_id": test_df.test_id,
"price": preds.reshape(-1),
})
 submission.to_csv("./rnn_ridge_submission_var_2.csv", index=False )
model.evaluate(X_test, y_test )<save_to_csv>
stop_real = datetime.now()
execution_time_real = stop_real-start_real
print(execution_time_real )