Kaggle competition: Avito Demand Prediction Challenge
<SOS> metric: RMSE Kaggle data source: avito-demand-prediction-challenge<import_modules>
%matplotlib inline
DATA_DIR = '.. /input/avito-demand-prediction/'
EMB_PATH = '.. /input/fasttest-common-crawl-russian/cc.ru.300.vec'
target_col = 'deal_probability'
max_features = 50000
maxlen = 100
embed_size = 300
os.listdir(DATA_DIR )
from kaggle.competitions import twosigmanews
import time
import numpy as np
import pandas as pd
import sklearn
import matplotlib.pyplot as plt<define_variables>
usecols = ['region', 'city', 'parent_category_name', 'category_name', 'param_1',
'param_2', 'param_3', 'title', 'description', 'price', 'activation_date',
'item_seq_number', 'user_type', 'image_top_1']
train = pd.read_csv(DATA_DIR+'train.csv', usecols=usecols+[target_col])
 test = pd.read_csv(DATA_DIR+'test.csv', usecols=usecols )
env = twosigmanews.make_env()
days = env.get_prediction_days()<train_model>
train = train[train['activation_date']<'2017-03-29']
test.loc[test['activation_date']>'2017-04-18', 'activation_date'] = '2017-04-18'
( market_train, news_train)= env.get_training_data()<data_type_conversions>
%%time
train['na_cnt'] = train.isnull().sum(axis=1)
 train['na_price'] = train['price'].isnull().astype('int8')
 test['na_cnt'] = test.isnull().sum(axis=1)
 test['na_price'] = test['price'].isnull().astype('int8')
 train['item_seq_number_log1p'] = train['item_seq_number'].apply(lambda x:np.log1p(x))
test['item_seq_number_log1p'] = test['item_seq_number'].apply(lambda x:np.log1p(x))
train['image_top_1'] = train['image_top_1'].fillna(train['image_top_1'].max() +1)
 test['image_top_1'] = test['image_top_1'].fillna(test['image_top_1'].max() +1)
 size_cols = ['region', 'city', 'parent_category_name', 'category_name', 'image_top_1']
mean_cols = ['parent_category_name', 'category_name', 'image_top_1']
def add_group_size(df, by, y='price'):
grp = df.groupby(by)[y].size().map(lambda x:np.log1p(x))
grp = grp.rename('size_'+'_'.join(by)).reset_index()
df = df.merge(grp, on=by, how='left')
 return df
def add_group_mean(df, by, y='price'):
grp = df.groupby(by)[y].mean()
grp = grp.rename('mean_price_'+'_'.join(by)).reset_index()
df = df.merge(grp, on=by, how='left')
 return df
for c in size_cols:
print('adding size by', [c, 'activation_date'])
 train = add_group_size(train, [c, 'activation_date'])
 test = add_group_size(test, [c, 'activation_date'])
 for c in mean_cols:
print('adding mean price by',(c,))
train = add_group_mean(train,(c,))
test = add_group_mean(test,(c,))
del train['activation_date'], test['activation_date']; gc.collect()
def process(market_df, newsdf, train=False):
market_df['time'] = market_df.time.dt.strftime("%Y%m%d" ).astype(int)
 cat_cols = ['assetCode']
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10']
market_df['bartrend'] = market_df['close'] / market_df['open']
market_df['average'] =(market_df['close'] + market_df['open'])/2
market_df['pricevolume'] = market_df['volume'] * market_df['close']
newsdf['time'] = newsdf.time.dt.strftime("%Y%m%d" ).astype(int)
 newsdf['assetCode'] = newsdf['assetCodes'].map(lambda x: list(eval(x)) [0])
 newsdf['position'] = newsdf['firstMentionSentence'] / newsdf['sentenceCount']
newsdf['coverage'] = newsdf['sentimentWordCount'] / newsdf['wordCount']
droplist = ['sourceTimestamp','firstCreated','sourceId','headline','takeSequence','provider','firstMentionSentence',
'sentenceCount','bodySize','headlineTag','marketCommentary','subjects','audiences','sentimentClass',
'assetName', 'assetCodes','urgency','wordCount','sentimentWordCount']
newsdf.drop(droplist, axis=1, inplace=True)
 market_df.drop(['assetName', 'volume'], axis=1, inplace=True)
 newsgp = newsdf.groupby(['time','assetCode'], sort=False ).aggregate(np.mean ).reset_index()
market_train = pd.merge(market_df, newsgp, how='left', on=['time', 'assetCode'], copy=False)
 num_cols = [x for x in market_train.columns if x not in cat_cols +['returnsOpenNextMktres10', 'universe']]
if train:
print(f"In {(market_train['bartrend'] >= 1.2 ).sum() } lines price increased by 20% or more...dropping")
 market_train = market_train[market_train['bartrend'] <= 1.2]
print(f"In {(market_train['bartrend'] <= 0.8 ).sum() } lines price decreased by 20% or more...dropping")
 market_train = market_train[market_train['bartrend'] >= 0.8]
market_train[num_cols] = market_train[num_cols].fillna(0)
 return market_train, cat_cols, num_cols
market_train, cat_cols, num_cols = process(market_train, news_train, train=True)
 scaler = StandardScaler()
market_train[num_cols] = scaler.fit_transform(market_train[num_cols])
 train_indices, val_indices = train_test_split(market_train.index.values,test_size=0.25, random_state=23)
 def encode(encoder, x):
len_encoder = len(encoder)
 try:
id = encoder[x]
except KeyError:
id = len_encoder
return id
encoders = [{} for cat in cat_cols]
for i, cat in enumerate(cat_cols):
print('encoding %s...' % cat, end=' ')
 encoders[i] = {l: id for id, l in enumerate(market_train.loc[train_indices, cat].astype(str ).unique())}
market_train[cat] = market_train[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
print('Done')
 embed_sizes = [len(encoder)+ 1 for encoder in encoders]
market_train[num_cols] = market_train[num_cols].fillna(0)
 print('scaling numerical columns' )<choose_model_class>
%%time
size_cols = [c for c in train.columns if 'size_' in c]
mean_cols = [c for c in train.columns if 'mean_' in c]
for c in ['price'] + mean_cols:
train[c] = train[c].fillna(-1)
 test[c] = test[c].fillna(-1)
 train[c] = train[c].apply(lambda x:np.log1p(x))
test[c] = test[c].apply(lambda x:np.log1p(x))
train = train.fillna('неизвестный')
 test = test.fillna('неизвестный')
 y = train[target_col].values
del train[target_col]; gc.collect()
train_num = len(train)
 str_cols = [c for c in train.columns if c not in [
'na_price', 'na_cnt', 'item_seq_number_log1p',
'price', 'item_seq_number', 'image_top_1', 'user_type', target_col] + size_cols + mean_cols
]
cat_cols = ['region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3',
'item_seq_number', 'user_type', 'image_top_1']
df = pd.concat([train, test], ignore_index=True)
 del train, test; gc.collect()
df['text'] = ''
for i, c in enumerate(df.columns):
if c in str_cols:
df['text'] += ' ' + df[c].str.lower()
if c not in cat_cols:
del df[c]; gc.collect()
if c in cat_cols:
df[c] = pd.factorize(df[c])[0]
categorical_inputs = []
for cat in cat_cols:
categorical_inputs.append(Input(shape=[1], name=cat))
categorical_embeddings = []
for i, cat in enumerate(cat_cols):
categorical_embeddings.append(Embedding(embed_sizes[i], 10 )(categorical_inputs[i]))
categorical_logits = Flatten()(categorical_embeddings[0])
 categorical_logits = Dense(32,activation='relu' )(categorical_logits)
 numerical_inputs = Input(shape=(31,), name='num')
 numerical_logits = numerical_inputs
numerical_logits = BatchNormalization()(numerical_logits)
 numerical_logits = Dense(512,activation='sigmoid' )(numerical_logits)
 numerical_logits = Dense(256,activation='sigmoid' )(numerical_logits)
 numerical_logits = Dense(128,activation='sigmoid' )(numerical_logits)
 numerical_logits = Dense(64,activation='sigmoid' )(numerical_logits)
 logits = Concatenate()([numerical_logits,categorical_logits])
 logits = Dense(64,activation='sigmoid' )(logits)
 out = Dense(1, activation='sigmoid' )(logits)
 model = Model(inputs = categorical_inputs + [numerical_inputs], outputs=out)
 model.compile(optimizer='adam',loss=binary_crossentropy )<prepare_x_and_y>
print(df.info())
 for c in cat_cols+['na_cnt']:
if df[c].max() <2**7:
df[c] = df[c].astype('int8')
 elif df[c].max() <2**15:
df[c] = df[c].astype('int16')
 elif df[c].max() <2**31:
df[c] = df[c].astype('int32')
 else:
continue
print(df.info() )
def get_input(market_train, indices):
X_num = market_train.loc[indices, num_cols].values
X = {'num':X_num}
for cat in cat_cols:
X[cat] = market_train.loc[indices, cat_cols].values
y =(market_train.loc[indices,'returnsOpenNextMktres10'] >= 0 ).values
r = market_train.loc[indices,'returnsOpenNextMktres10'].values
u = market_train.loc[indices, 'universe']
d = market_train.loc[indices, 'time']
return X,y,r,u,d
X_train,y_train,r_train,u_train,d_train = get_input(market_train, train_indices)
 X_valid,y_valid,r_valid,u_valid,d_valid = get_input(market_train, val_indices )<train_model>
df['char_len'] = df['text'].apply(lambda x:np.log1p(len(x)))
 df['char_len'] = df['char_len'].astype('float32')
 df['word_len'] = df['text'].apply(lambda x:np.log1p(len(x.split(' '))))
df['word_len'] = df['word_len'].astype('float32' )
check_point = ModelCheckpoint('model.hdf5',verbose=True, save_best_only=True)
 early_stop = EarlyStopping(patience=3,verbose=True)
 model.fit(X_train,y_train.astype(int),
validation_data=(X_valid,y_valid.astype(int)) ,
epochs=15,
verbose=True,
callbacks=[early_stop,check_point] )<categorify>
%%time
print('tokenizing...')
 tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(df['text'].values.tolist() )
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
predicted_confidences = np.array([])
 for(market_obs_df, news_obs_df, predictions_template_df)in days:
t = time.time()
market_obs_df, cat_cols, num_cols = process(market_obs_df, news_obs_df)
 market_obs_df[num_cols] = scaler.transform(market_obs_df[num_cols])
 market_obs_df['assetCode_encoded'] = market_obs_df[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
X_num_test = market_obs_df[num_cols].values
X_test = {'num':X_num_test}
X_test['assetCode'] = market_obs_df['assetCode_encoded'].values
prep_time += time.time() - t
t = time.time()
market_prediction = model.predict(X_test)[:,0]*2 -1
predicted_confidences = np.concatenate(( predicted_confidences, market_prediction))
prediction_time += time.time() -t
t = time.time()
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':market_prediction})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
total = prep_time + prediction_time + packaging_time
print(f'Preparing Data: {prep_time:.2f}s')
 print(f'Making Predictions: {prediction_time:.2f}s')
 print(f'Packing: {packaging_time:.2f}s')
 print(f'Total: {total:.2f}s' )<save_to_csv>
%%time
def fill_rand_norm(embedding_matrix):
emb_zero_shape = embedding_matrix[embedding_matrix==0].shape
emb_non_zero_mean = embedding_matrix[embedding_matrix!=0.].mean()
emb_non_zero_std = embedding_matrix[embedding_matrix!=0.].std()
embedding_matrix[embedding_matrix==0] = np.random.normal(emb_non_zero_mean,
emb_non_zero_std,
emb_zero_shape)
 return embedding_matrix
embedding_matrix = fill_rand_norm(embedding_matrix )
env.write_submission_file()<train_model>
%%time
text = df['text'].values
del df['text']; gc.collect()
text = tokenizer.texts_to_sequences(text)
 text = sequence.pad_sequences(text, maxlen=maxlen )
env = twosigmanews.make_env() 
(market_train, news_train)= env.get_training_data()<count_duplicates>
%%time
df_train = df[:train_num]
text_train = text[:train_num]
df_test = df[train_num:]
text_test = text[train_num:]
del text, df; gc.collect()
<feature_engineering>
%%time
def get_keras_data(df, text):
X = {}
for c in df.columns:
X[c] = df[c].values
X['text'] = text
return X
X_train = get_keras_data(df_train, text_train)
 X_test = get_keras_data(df_test, text_test)
 del df_train, text_train, df_test, text_test; gc.collect()
market_train = market_train[market_train.returnsOpenNextMktres10<1][market_train.returnsOpenNextMktres10>-1] 
<feature_engineering>
BatchNormalization, SpatialDropout1D, GlobalAveragePooling1D, GRU, Bidirectional, GlobalMaxPooling1D, Conv1D

 def dot_product(x, kernel):

 if K.backend() == 'tensorflow':
return K.squeeze(K.dot(x, K.expand_dims(kernel)) , axis=-1)
 else:
return K.dot(x, kernel)
 class AttentionWithContext(Layer):

 def __init__(self,
W_regularizer=None, u_regularizer=None, b_regularizer=None,
W_constraint=None, u_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.u_regularizer = regularizers.get(u_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.u_constraint = constraints.get(u_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
super(AttentionWithContext, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1], input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 if self.bias:
self.b = self.add_weight(( input_shape[-1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 self.u = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_u'.format(self.name),
regularizer=self.u_regularizer,
constraint=self.u_constraint)
 super(AttentionWithContext, self ).build(input_shape)
 def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
uit = dot_product(x, self.W)
 if self.bias:
uit += self.b
uit = K.tanh(uit)
 ait = dot_product(uit, self.u)
 a = K.exp(ait)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], input_shape[-1]
def calcRsi(series, period = 14):

 try:
delta = series.diff().dropna()
u = delta * 0
d = u.copy()
u[delta > 0] = delta[delta > 0]
d[delta < 0] = -delta[delta < 0]
u[u.index[period-1]] = np.mean(u[:period])
 u = u.drop(u.index[:(period-1)])
 d[d.index[period-1]] = np.mean(d[:period])
 d = d.drop(d.index[:(period-1)])
 rs = u.ewm(com=period-1, adjust=False ).mean() \
/ d.ewm(com=period-1, adjust=False ).mean()
rsi = 100 - 100 /(1 + rs)
 except IndexError:
rsi = 0
return rsi<feature_engineering>
emb_cat_max = {}
for c in cat_cols:
emb_cat_max[c] = max(X_train[c].max() , X_test[c].max())+1
params = {}
params['maxlen'] = maxlen
params['nb_words'] = nb_words
params['embedding_matrix'] = embedding_matrix
params['word_emb_size'] = embed_size
params['text_emb_dropout'] = 0.2
params['n_rnn'] = 64
params['emb_cat_max'] = emb_cat_max
params['emb_size'] = 32
params['n_output'] = 1
params['use_fm'] = True
params['use_deep'] = True
params['use_rnn'] = True
params['use_cnn'] = True
params['use_att'] = False
params['use_batch_norm'] = False
params['cnn_param'] = dict(filters=192, kernel_size=3)
 params['deep_layers'] = [256, 256, 256]
params['drop_out'] = [0.5, 0.5, 0.5]
params['output_drop_out'] = 0.2
assert len(params['drop_out'])==len(params['deep_layers'])
 params['cat_feats'] = cat_cols
params['num_feats'] = size_cols+mean_cols+['price', 'char_len', 'word_len', 'na_price', 'na_cnt', 'item_seq_number_log1p']
params['lr'] = 0.001
params['decay'] = 1e-6
def addBollinger(df, period=20, col='close'):

 bbmid_series = df[col].rolling(window=period ).mean()
series_stdev = df[col].rolling(window=period ).std()
df['BBUpperBand'] = bbmid_series + 2*series_stdev
df['BBLowerBand'] = bbmid_series - 2*series_stdev
df['BBBandwidth'] = df['BBUpperBand'] - df['BBLowerBand']
df['BBMiddleBand'] = bbmid_series
return df<feature_engineering>
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)))

def get_model(params=params):
cats = [Input(shape=[1], name=name)for name in params['cat_feats']]
nums = [Input(shape=[1], name=name)for name in params['num_feats']]
emb_fn = lambda name: Embedding(params['emb_cat_max'][name], params['emb_size'])
 embs = []
for name, cat in zip(params['cat_feats'], cats):
embs.append(emb_fn(name )(cat))
texts = Input(shape=(params['maxlen'],), name='text')
 text_emb = Embedding(params['nb_words'],
params['word_emb_size'],
weights=[params['embedding_matrix']],
name='text_emb' )(texts)
 outs = []
if params['use_rnn']:
x_text = SpatialDropout1D(params['text_emb_dropout'] )(text_emb)
 x_rnn = Bidirectional(GRU(params['n_rnn'], return_sequences=True))(x_text)
 avg_pool_rnn = GlobalAveragePooling1D()(x_rnn)
 max_pool_rnn = GlobalMaxPooling1D()(x_rnn)
 outs += [avg_pool_rnn, max_pool_rnn]
if params['use_cnn']:
x_text = SpatialDropout1D(params['text_emb_dropout'] )(text_emb)
 x_cnn = Conv1D(**params['cnn_param'] )(x_text)
 avg_pool_cnn = GlobalAveragePooling1D()(x_cnn)
 max_pool_cnn = GlobalMaxPooling1D()(x_cnn)
 outs += [avg_pool_cnn, max_pool_cnn]
if params['use_att']:
x_text = SpatialDropout1D(params['text_emb_dropout'] )(text_emb)
 x_att = AttentionWithContext()(x_text)
 outs += [x_att]
if params['use_rnn']:
x_rnn_att = AttentionWithContext()(x_rnn)
 outs += [x_rnn_att]
if params['use_cnn']:
x_cnn_att = AttentionWithContext()(x_cnn)
 outs += [x_cnn_att]
if params['use_fm']:
first_order = [Flatten()(emb0)for emb0 in embs]
second_order = []
for emb1, emb2 in combinations(embs, 2):
dot_layer = dot([Flatten()(emb1), Flatten()(emb2)], axes=1)
 second_order.append(dot_layer)
 first_order = add(first_order)
 second_order = add(second_order)
 outs += [first_order, second_order]
if params['use_deep']:
all_in = [Flatten()(emb)for emb in embs] + nums + [Flatten()(text_emb)]
x_in = concatenate(all_in)
 for idx,(drop_p, num_dense)in enumerate(zip(params['drop_out'], params['deep_layers'])) :
x_in = Dense(num_dense, activation='relu' )(x_in)
 if params['use_batch_norm']:
x_in =(BatchNormalization() )(x_in)
 else:
x_in = Dropout(drop_p )(x_in)
 deep = x_in
outs += [deep]
total_out = concatenate(outs)if len(outs)>1 else outs[0]
if 0<params['output_drop_out']<1:
total_out = Dropout(params['output_drop_out'] )(total_out)
 output = Dense(params['n_output'], activation='sigmoid' )(total_out)
 model = Model(inputs=cats+nums+[texts], output=output)
 optimizer = Adam(lr=params['lr'], decay=params['decay'])
 model.compile(loss=root_mean_squared_error,
optimizer=optimizer,
metrics=[root_mean_squared_error])
 return model
model = get_model()
def addMACD(df):
ema_fast = df['close'].ewm(span=12 ).mean()
ema_slow = df['close'].ewm(span=26 ).mean()
signal_line = df['close'].ewm(span=9 ).mean()
df['macd'] = ema_fast - ema_slow
df['macd_signal'] = df.macd.ewm(span=9, adjust=False ).mean()
df['macdh'] = df['macd'] - df['macd_signal']
return df<feature_engineering>
file_path = "model.hdf5"
check_point = ModelCheckpoint(file_path, monitor='val_root_mean_squared_error', mode='min',
save_best_only=True, verbose=1)
 early_stop = EarlyStopping(monitor='val_root_mean_squared_error', patience=2, mode='min')
 batch_size = 2**10 * 2
epochs = 10
sample_weight = np.ones(y.shape)
 sample_weight[y<1e-7] = 1 + len(y[y<1e-7])/len(y)
 history = model.fit(X_train, y, sample_weight=sample_weight,
batch_size=batch_size, epochs=epochs,
validation_split=0.05, verbose=1,
callbacks=[check_point, early_stop] )
def allindicator(market_train):
market_train['RSI'] = calcRsi(market_train['close'], 14)
 market_train = addBollinger(market_train)
 market_train = addMACD(market_train)
 return market_train<import_modules>
model.load_weights(file_path )
from multiprocessing import Pool 
<groupby>
pred = model.predict(X_test, batch_size=batch_size, verbose=1 )
def groupAsset(market_train):
all_df = []
df_codes = market_train.groupby('assetCode')
 df_codes = [df_code[1] for df_code in df_codes]
pool = Pool(4)
 all_df = pool.map(allindicator, df_codes)
 new_df = pd.concat(all_df)
 pool.close()
del all_df, df_codes, market_train
return new_df<feature_engineering>
sub = pd.read_csv(DATA_DIR+'sample_submission.csv' )
def data_prep(market_train,news_train):
market_train = groupAsset(market_train)
 market_train['formatdate']=market_train['time'].dt.date
news_train['formatdate'] = news_train.firstCreated.dt.date
for col_cat in ['headlineTag', 'provider', 'sourceId']:
news_train[col_cat], uniques = pd.factorize(news_train[col_cat])
 del uniques
news_train_sim = news_train.drop(columns=['time','sourceTimestamp','firstCreated','headline','subjects','audiences','assetCodes'])
 news_train_1 = news_train_sim.groupby(by=['assetName','formatdate'],as_index=False,observed=True ).mean().add_suffix('_mean')
 news_train_1.rename(columns={'assetName_mean': 'assetName', 'formatdate_mean': 'formatdate'}, inplace=True)
 merge_train = pd.merge(market_train, news_train_1, how='left', left_on=['formatdate', 'assetName'],
right_on=['formatdate', 'assetName'])
 merge_train.fillna(0,inplace=True)
 del market_train, news_train_sim, news_train_1
return merge_train 
<categorify>
sub[target_col] = pred
sub.head()
merge_train = data_prep(market_train,news_train)
 up = merge_train.returnsOpenNextMktres10 >= 0
up = up.values
fcol = [c for c in merge_train.columns if c not in ['assetCode','assetName','time',
'returnsOpenNextMktres10', 'formatdate', 'universe']]
X = merge_train[fcol].values
r = merge_train.returnsOpenNextMktres10.values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng )<split>
scr = min(history.history['val_root_mean_squared_error'])
 print('save to '+f'nn_{scr}.csv')
 sub.to_csv(f'nn_{scr}.csv', index=False )