Kaggle competition: Predict Future Sales
data = data.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 print(data.shape, test.shape)
 data = data.merge(right=previous_application.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=previous_application.reset_index() , how='left', on='SK_ID_CURR')
 print(data.shape, test.shape)
 data = data.merge(right=POS_CASH_balance.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=POS_CASH_balance.reset_index() , how='left', on='SK_ID_CURR')
 print(data.shape, test.shape)
 data = data.merge(right=installments_payments.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=installments_payments.reset_index() , how='left', on='SK_ID_CURR')
 print(data.shape, test.shape)
 gc.collect()<count_missing_values>
categories = pd.read_csv(".. /input/eng-translations/categories_eng.csv")
 items = pd.read_csv(".. /input/eng-translations/items_eng.csv")
 sales = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sales_train.csv")
 test = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 shops = pd.read_csv(".. /input/eng-translations/shops_eng.csv")
 submission = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sample_submission.csv" )
print('Removing features with more than 80% missing...')
 test = test[test.columns[data.isnull().mean() < 0.80]]
data = data[data.columns[data.isnull().mean() < 0.80]]
print(data.shape, test.shape )<split>
def downcast1(df, verbose=True):

 start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
dtype_name = df[col].dtype.name
if dtype_name == 'object':
pass
elif dtype_name == 'bool':
df[col] = df[col].astype('int8')
 elif dtype_name.startswith('int')or(df[col].round() == df[col] ).all() :
df[col] = pd.to_numeric(df[col], downcast='integer')
 else:
df[col] = pd.to_numeric(df[col], downcast='float')
 end_mem = df.memory_usage().sum() / 1024**2
if verbose:
print('{:.1f}% compressed'.format(100 *(start_mem - end_mem)/ start_mem))
return df
gc.enable()
folds = KFold(n_splits=4, shuffle=True, random_state=546789)
 oof_preds = np.zeros(data.shape[0])
 sub_preds = np.zeros(test.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in data.columns if f not in ['SK_ID_CURR']]
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(data)) :
trn_x, trn_y = data[feats].iloc[trn_idx], y.iloc[trn_idx]
val_x, val_y = data[feats].iloc[val_idx], y.iloc[val_idx]
clf = LGBMClassifier(
 n_estimators=10000,
learning_rate=0.03,
num_leaves=34,
colsample_bytree=0.9,
subsample=0.8,
max_depth=8,
reg_alpha=.1,
reg_lambda=.1,
min_split_gain=.01,
min_child_weight=300,
silent=-1,
verbose=-1, 
)
 clf.fit(trn_x, trn_y,
eval_set= [(trn_x, trn_y),(val_x, val_y)],
eval_metric='auc', verbose=100, early_stopping_rounds=100 
)
 oof_preds[val_idx] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))
 del clf, trn_x, trn_y, val_x, val_y
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(y, oof_preds))
test['TARGET'] = sub_preds
test[['SK_ID_CURR', 'TARGET']].to_csv('submission1LGBM.csv', index=False)
 cols = feature_importance_df[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)];<import_modules>
def cleans(i):

 pattern = r'[A-Za-z0-9]+'
finds = re.findall(pattern, str(i))
stringy = ""
for j in finds:
stringy += f" {j}"
return stringy
from fastai.imports import *
from fastai.structured import *
from fastai.column_data import *
from torch.nn import functional as F
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split<load_from_disk>
shops["clean"] = shops["shop_name"].apply(cleans)
 shops.head()
df_train = pd.read_feather('.. /input/home-credit-data-processing-for-neural-networks/tables_merged_train')
 df_test = pd.read_feather('.. /input/home-credit-data-processing-for-neural-networks/tables_merged_test' )<count_values>
sales.loc[sales["shop_id"]==0, "shop_id"] = 57
sales.loc[sales["shop_id"]==1, "shop_id"] = 58
sales.loc[sales["shop_id"]==10, "shop_id"] = 11
sales.loc[sales["shop_id"]==39, "shop_id"] = 40
test.loc[test['shop_id'] == 0, 'shop_id'] = 57
test.loc[test['shop_id'] == 1, 'shop_id'] = 58
test.loc[test['shop_id'] == 10, 'shop_id'] = 11
test.loc[test['shop_id'] == 39, 'shop_id'] = 40
df_train.dtypes.value_counts()<drop_column>
unique_test_shops = test["shop_id"].unique()
sales = sales[sales["shop_id"].isin(unique_test_shops)]
print(f"Number of Unique Shops in Test Data:{len(unique_test_shops)}
Number of Unique Shops in Sales Data:{len(sales['shop_id'].unique())}" )
cat_vars = [col for col in df_train if df_train[col].dtype.name != 'float64' and df_train[col].dtype.name != 'float32' and len(df_train[col].unique())< 150]
cat_vars.remove('TARGET' )<count_unique_values>
shops.drop("shop_name", axis=1, inplace=True )
cat_sz = [(c, len(df_train[c].unique())+1)for c in cat_vars]<split>
shops["city"] = shops["clean"].apply(lambda x: x.split() [0] )
y = np.array(df_train['TARGET'])
 df_train.drop('TARGET', axis = 1, inplace=True)
 df_to_nn_train, df_to_nn_valid, y_train, y_valid = train_test_split(df_train, y, test_size=0.33, random_state=23, stratify = y )<categorify>
le = LabelEncoder()
shops["city"] = le.fit_transform(shops["city"])
 shops.drop("clean", axis=1, inplace=True)
def preprocess_fast_ai(df_to_nn_train, df_to_nn_valid, cat_vars):
for v in cat_vars: df_to_nn_train[v] = df_to_nn_train[v].astype('category' ).cat.as_ordered()
apply_cats(df_to_nn_valid, df_to_nn_train)
 df, _, nas, mapper = proc_df(df_to_nn_train, do_scale=True, skip_flds=['SK_ID_CURR'])
 df_valid, _, nas, mapper = proc_df(df_to_nn_valid, do_scale=True, na_dict=nas, mapper=mapper, skip_flds=['SK_ID_CURR'])
 return df, df_valid<normalization>
items["item_name"] = items["item_name"].str.lower()
items["item_name_clean"] = items["item_name"].apply(cleans)
 items.drop("item_name", axis=1, inplace=True )
%time df, df_valid = preprocess_fast_ai(df_to_nn_train, df_to_nn_valid, cat_vars )<define_variables>
items["item_name_five"] = [x[:5] for x in items["item_name_clean"]]
items["item_name_five"] = le.fit_transform(items["item_name_five"])
 items.drop("item_name_clean", axis=1, inplace=True )
emb_szs = [(c, min(50,(c+1)//2)) for _,c in cat_sz]<data_type_conversions>
items["first_sale_date"] = sales.groupby("item_id" ).agg({"date_block_num":"min"})["date_block_num"]
items
md = ColumnarModelData.from_data_frames('', trn_df = df, val_df = df_valid,
trn_y = y_train.astype('int'), val_y = y_valid.astype('int'),
cat_flds=cat_vars, bs=512, is_reg= False )<choose_model_class>
items[items["first_sale_date"].isna() ]
items["first_sale_date"] = items["first_sale_date"].fillna(34 )
class MixedInputModel(nn.Module):
def __init__(self, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
y_range=None, use_bn=False, is_reg=True, is_multi=False):
super().__init__()
self.embs = nn.ModuleList([nn.Embedding(c, s)for c,s in emb_szs])
 for emb in self.embs: emb_init(emb)
 n_emb = sum(e.embedding_dim for e in self.embs)
 self.n_emb, self.n_cont= n_emb, n_cont
szs = [n_emb + n_cont] + szs
self.lins = nn.ModuleList([
nn.Linear(szs[i], szs[i+1])for i in range(len(szs)-1)])
 self.bns = nn.ModuleList([
nn.BatchNorm1d(sz)for sz in szs[1:]])
 for o in self.lins: kaiming_normal(o.weight.data)
 self.outp = nn.Linear(szs[-1], out_sz)
 kaiming_normal(self.outp.weight.data)
 self.emb_drop = nn.Dropout(emb_drop)
 self.drops = nn.ModuleList([nn.Dropout(drop)for drop in drops])
 self.bn = nn.BatchNorm1d(n_cont)
 self.use_bn,self.y_range = use_bn,y_range
self.is_reg = is_reg
self.is_multi = is_multi
def forward(self, x_cat, x_cont):
x = []
for i,e in enumerate(self.embs):
x.append(e(x_cat[:,i]))
x = torch.cat(x, 1)
 x = self.emb_drop(x)
 x2 = self.bn(x_cont)
 x = torch.cat([x, x2], 1)
 for l,d,b in zip(self.lins, self.drops, self.bns):
x = F.relu(l(x))
if self.use_bn: x = b(x)
 x = d(x)
 x = self.outp(x)
 x = F.log_softmax(x)
 return x<choose_model_class>
categories["category"] = categories["category_name"].apply(lambda x: x.split() [0])
 categories
m = MixedInputModel(emb_szs, n_cont = len(df.columns)-len(cat_vars),
emb_drop = 0.05, out_sz = 2, szs = [500, 250, 250], drops = [0.1, 0.1, 0.1],
y_range = None, use_bn = False, is_reg = False, is_multi = False)
 bm = BasicModel(m.cuda() , 'binary_classifier' )<choose_model_class>
categories["category"].value_counts()
class StructuredLearner(Learner):
def __init__(self, data, models, **kwargs):
super().__init__(data, models, **kwargs)
 self.crit = F.nll_loss
learn = StructuredLearner(md, bm )<set_options>
categories.loc[categories["category"] == "Game"] = "Games"
learn.lr_find(1e-4, 1)
 learn.sched.plot(100 )<train_model>
def make_misc(x):

 if len(categories[categories['category']==x])>= 5:
return x
else:
return 'Misc'
categories["cats"] = categories["category"].apply(make_misc)
 categories
lr = 1e-1
learn.fit(lr, 3, metrics=[roc_auc_own] )<predict_on_test>
categories.drop(["category", "category_name"], axis=1, inplace=True )
logpreds = learn.predict()
preds = np.exp(logpreds[:,1] )<compute_test_metric>
categories["cats_le"] = le.fit_transform(categories["cats"])
 categories.drop("cats", inplace=True, axis=1 )
print(classification_report(y_valid,
preds_binary,
target_names= ['0', '1']))<data_type_conversions>
sales = sales[sales["item_price"] > 0]
sales = sales[sales["item_price"] < 50000]
sales = sales[sales["item_cnt_day"] > 0]
sales = sales[sales["item_cnt_day"] < 1000]
sales["item_price"] = sales["item_price"].apply(lambda x: round(x,2))
sales
class ColumnarDataset(Dataset):
def __init__(self, cats, conts, y, is_reg, is_multi):
n = len(cats[0])if cats else len(conts[0])
 self.cats = np.stack(cats, 1 ).astype(np.int64)if cats else np.zeros(( n,1))
self.conts = np.stack(conts, 1 ).astype(np.float32)if conts else np.zeros(( n,1))
self.y = np.zeros(( n,1)) if y is None else y
if is_reg:
self.y = self.y[:,None]
self.is_reg = is_reg
self.is_multi = is_multi
def __len__(self): return len(self.y)
 def __getitem__(self, idx):
return [self.cats[idx], self.conts[idx], self.y[idx]]
@classmethod
def from_data_frames(cls, df_cat, df_cont, y=None, is_reg=True, is_multi=False):
cat_cols = [c.values for n,c in df_cat.items() ]
cont_cols = [c.values for n,c in df_cont.items() ]
return cls(cat_cols, cont_cols, y, is_reg, is_multi)
 @classmethod
def from_data_frame(cls, df, cat_flds, y=None, is_reg=False, is_multi=False):
return cls.from_data_frames(df[cat_flds], df.drop(cat_flds, axis=1), y, is_reg, is_multi)
 class ColumnarModelData(ModelData):
def __init__(self, path, trn_ds, val_ds, bs, test_ds=None, shuffle=True):
test_dl = DataLoader(test_ds, bs, shuffle=False, num_workers=1)if test_ds is not None else None
super().__init__(path, DataLoader(trn_ds, bs, shuffle=shuffle, num_workers=1),
DataLoader(val_ds, bs*2, shuffle=False, num_workers=1), test_dl)
 @classmethod
def from_data_frames(cls, path, trn_df, trn_y, cat_flds, bs, val_df = None, val_y = None, is_reg = False, is_multi = False, test_df=None):
trn_ds = ColumnarDataset.from_data_frame(trn_df, cat_flds, trn_y, is_reg, is_multi)
 val_ds = ColumnarDataset.from_data_frame(val_df, cat_flds, val_y, is_reg, is_multi)if val_df is not None else None
test_ds = ColumnarDataset.from_data_frame(test_df, cat_flds, None, is_reg, is_multi)if test_df is not None else None
return cls(path, trn_ds, val_ds, bs, test_ds=test_ds)
 @classmethod
def from_data_frame(cls, path, val_idxs, df, y, cat_flds, bs, is_reg=True, is_multi=False, test_df=None): 
(( val_df, trn_df),(val_y, trn_y)) = split_by_idx(val_idxs, df, y)
 return cls.from_data_frames(path, trn_df, val_df, trn_y, val_y, cat_flds, bs, is_reg, is_multi, test_df=test_df )<normalization>
group = sales.groupby(index_feats ).agg({"item_cnt_day": "sum"})
 group = group.reset_index()
group = group.rename(columns={"item_cnt_day": "item_cnt_month"})
 train = pd.merge(train, group, on=index_feats, how="left")
 train
train_ids = df_train['SK_ID_CURR']
test_ids = df_test['SK_ID_CURR']
%time train_df, test_df = preprocess_fast_ai(df_train, df_test, cat_vars )<train_model>
del group
gc.collect()
ros = RandomOverSampler()
df_resampled, y_resampled = ros.fit_sample(df, y_train)
 df_resampled = pd.DataFrame(df_resampled, columns = df.columns)
 y_valid.mean() , y_resampled.mean()<data_type_conversions>
group = sales.groupby(index_feats ).agg({"item_cnt_day":"count"})
 group = group.reset_index()
group = group.rename(columns={"item_cnt_day":"item_cnt"})
 train = pd.merge(train, group, on=index_feats, how="left")
 train.sample(5 )
md = ColumnarModelData.from_data_frames('', trn_df = df_resampled,
val_df = df_valid, trn_y = y_resampled.astype('int'),
val_y = y_valid.astype('int'), cat_flds=cat_vars, bs=1024, is_reg = False,
test_df = test_df)
 class StructuredLearner(Learner):
def __init__(self, data, models, **kwargs):
super().__init__(data, models, **kwargs)
 self.crit = F.nll_loss
m = MixedInputModel(emb_szs, n_cont = len(df.columns)-len(cat_vars),
emb_drop = 0.4, out_sz = 2, szs = [1000, 500],
drops = [0.6, 0.6],y_range = None, use_bn = False, is_reg = False)
 bm = BasicModel(m.cuda() , 'binary_classifier')
 learn = StructuredLearner(md, bm )<set_options>
del group, sales
gc.collect()
learn.lr_find(1e-2, 2)
 learn.sched.plot(100 )<train_model>
test["date_block_num"] = 34
all_data = pd.concat([train, test.drop("ID", axis=1)], ignore_index=True, keys=index_feats)
 all_data = all_data.fillna(0)
 all_data.sample(10 )
lr = 0.1
learn.fit(lr, 3, metrics=[roc_auc_own] )<train_model>
all_data = pd.merge(all_data, shops, on="shop_id", how="left")
 all_data = pd.merge(all_data, items, on="item_id", how="left")
 all_data = pd.merge(all_data, categories, on="category_id", how="left")
 all_data.sample(10 )
learn.fit(lr, 2, metrics=[roc_auc_own], cycle_len=1, cycle_mult=2 )<compute_train_metric>
all_data = downcast1(all_data)
 del shops, items, categories
gc.collect()
print(classification_report(y_valid,
preds_binary,
target_names= ['0', '1']))
false_positive_rate, true_positive_rate, threshold = roc_curve(y_valid,
preds_valid )<save_to_csv>
def add_mean_feats(df, mean_feats, index_features, agg_col="item_cnt_month", agg_func="mean"):

 if len(index_features)== 2:
feature_name = index_features[1] + f"_{agg_col}_{agg_func}"
else:
feature_name = index_features[1] + "_" + index_features[2] + f"_{agg_col}_{agg_func}"
group = df.groupby(index_features ).agg({agg_col:agg_func} ).reset_index().rename(columns={agg_col:feature_name})
 df = pd.merge(df, group, on=index_features, how="left")
 df = downcast1(df)
 mean_feats.append(feature_name)
 del group
gc.collect()
return df, mean_feats
logpreds = learn.predict(True)
 preds = np.exp(logpreds[:,1])
 submission = pd.DataFrame({'SK_ID_CURR': df_test['SK_ID_CURR'],
'TARGET': preds})
 submission.to_csv('submission.csv', index=False, float_format='%.8f' )<find_best_params>
item_mean_features = []
all_data, item_mean_features = add_mean_feats(all_data, item_mean_features, ["date_block_num", "item_id"])
 all_data
m=learn.model
m.cuda()<create_dataframe>
all_data, item_mean_features = add_mean_feats(all_data, item_mean_features, ["date_block_num", "item_id", "city"] )
def get_embeddings(embs, dataframe, ids, cat_vars):
embeddings = np.concatenate([to_np(embs[i](V(dataframe[cat_vars[i]])))for i in range(len(embs)) ], axis = 1)
 embedding_columns = ["embedding_"+str(i)for i in range(embeddings.shape[1])]
embedding_df = pd.DataFrame(embeddings, columns=embedding_columns)
 embedding_df = pd.concat([embedding_df, ids], axis = 1)
 return embedding_df<save_to_csv>
shop_mean_features = []
all_data, shop_mean_features = add_mean_feats(all_data, shop_mean_features, ["date_block_num", "shop_id", "category_id"] )
train_embeddings = get_embeddings(m.embs, train_df, train_ids, cat_vars)
 test_embeddings = get_embeddings(m.embs, test_df, test_ids, cat_vars)
 train_embeddings.to_csv('train_embeddings.csv', index=False)
 test_embeddings.to_csv('test_embeddings.csv', index=False )<set_options>
cat_mean_features = []
all_data, cat_mean_features = add_mean_feats(all_data, cat_mean_features, ["date_block_num", "category_id"] )
seed(639)
 set_random_seed(5944)
 seed(639)
 set_random_seed(5944)
 float_data = pd.read_csv(".. /input/train.csv", dtype={"acoustic_data": np.float32, "time_to_failure": np.float32} ).values
def extract_features(z):
return np.c_[z.mean(axis=1),
z.min(axis=1),
z.max(axis=1),
z.std(axis=1)]
def create_X(x, last_index=None, n_steps=150, step_length=1000):
if last_index == None:
last_index=len(x)
 assert last_index - n_steps * step_length >= 0
temp =(x[(last_index - n_steps * step_length):last_index].reshape(n_steps, -1)- 5)/ 3
return np.c_[extract_features(temp),
extract_features(temp[:, -step_length // 10:]),
extract_features(temp[:, -step_length // 100:]),
extract_features(temp[:, -step_length // 300:])]
n_features = create_X(float_data[0:150000] ).shape[1]
print("Our RNN is based on %i features"% n_features)
 def generator(data, min_index=0, max_index=None, batch_size=16, n_steps=150, step_length=1000):
if max_index is None:
max_index = len(data)- 1
while True:
rows = np.random.randint(min_index + n_steps * step_length, max_index, size=batch_size)
 samples = np.zeros(( batch_size, n_steps, n_features))
targets = np.zeros(batch_size,)
 for j, row in enumerate(rows):
samples[j] = create_X(data[:, 0], last_index=row, n_steps=n_steps, step_length=step_length)
 targets[j] = data[row - 1, 1]
yield samples, targets
batch_size = 32
second_earthquake = 50085877
float_data[second_earthquake, 1]
train_gen = generator(float_data, batch_size=batch_size, min_index=second_earthquake + 1)
 valid_gen = generator(float_data, batch_size=batch_size, max_index=second_earthquake )<train_on_grid>
all_data, cat_mean_features = add_mean_feats(all_data, cat_mean_features, ["date_block_num", "cats_le"] )
cb = [ModelCheckpoint("model.hdf5", save_best_only=True)]
model = Sequential()
model.add(CuDNNLSTM(units=50))
model.add(Dropout(0.2))
model.add(Dense(units=1))
model.summary()
model.compile(optimizer='rmsprop',loss='mae')
 history = model.fit_generator(train_gen,
steps_per_epoch=1000,
epochs=430,
verbose=2,
callbacks=cb,
validation_data=valid_gen,
validation_steps=200 
)
 model = model = load_model("model.hdf5" )<load_from_csv>
def add_lags(df, lag_features, index_features, lag_feature, lags=[1,2,3], clip=False):

 df_temp = df[index_features + [lag_feature]].copy()
for i in lags:
feat_name = lag_feature + "_lag" + str(i)
 df_temp.columns = index_features + [feat_name]
df_temp["date_block_num"] += i
df = pd.merge(df, df_temp.drop_duplicates() , on=index_features, how="left")
 df[feat_name] = df[feat_name].fillna(0)
 if clip:
lag_feats_to_clip.append(feat_name)
 df = downcast1(df)
 del df_temp
gc.collect()
return df, lag_feats_to_clip
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id', dtype={"time_to_failure": np.float32})
 x = None
for i, seg_id in enumerate(tqdm(submission.index)) :
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x = seg['acoustic_data'].values[:]
submission.time_to_failure[i] =(model.predict(np.expand_dims(create_X(x), 0)))
 submission.to_csv('submission.csv' )<prepare_x_and_y>
lag_feats_to_clip = []
index_features = ["date_block_num", "shop_id", "item_id"]
all_data, lag_feats_to_clip = add_lags(all_data, lag_feats_to_clip, index_features, "item_cnt_month", clip=True)
 all_data, lag_feats_to_clip = add_lags(all_data, lag_feats_to_clip, index_features, "item_cnt", clip=True)
 all_data.sample(10 )
tr_data = pd.read_csv(".. /input/train.csv", iterator=True, chunksize=600_000)
 x_tr_data = pd.DataFrame()
y_tr_data = pd.Series()
for data in tr_data:
add_tr = new_column(data['acoustic_data'])
 x_tr_data = x_tr_data.append(add_tr, ignore_index=True)
 y_tr_data = y_tr_data.append(pd.Series(data['time_to_failure'].values[-1]))
del tr_data 
<train_model>
X_test_temp = all_data[all_data["date_block_num"]==34]
X_test_temp[item_mean_features].sum()
train_pool = Pool(x_tr_data, y_tr_data)
 my_model = CatBoostRegressor(iterations=1000, loss_function='MAE', boosting_type='Ordered')
 my_model.fit(x_tr_data, y_tr_data, silent=True)
 my_model.best_score_<save_to_csv>
for item in item_mean_features:
all_data, lag_feats_to_clip = add_lags(all_data, lag_feats_to_clip, index_features, item, clip=True )
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 test_data = pd.DataFrame(columns=x_tr_data.columns, dtype=np.float64, index=submission.index)
 for seg_id in test_data.index:
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 add_tr = new_column(seg['acoustic_data'])
 test_data.loc[seg_id]= add_tr
submission['time_to_failure'] = my_model.predict(test_data)
 submission.to_csv('submission.csv' )<set_options>
for shop in shop_mean_features:
all_data, lag_feats_to_clip = add_lags(all_data, lag_feats_to_clip, ["date_block_num", "shop_id", "category_id"],
shop, clip=True)
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
<train_on_grid>
for cat in cat_mean_features:
all_data, lag_feats_to_clip = add_lags(all_data, lag_feats_to_clip, ["date_block_num", "category_id"], cat, lags=[1,2,3], clip=True)
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta
def calc_change_rate(x):
change =(np.diff(x)/ x[:-1] ).values
change = change[np.nonzero(change)[0]]
change = change[~np.isnan(change)]
change = change[change != -np.inf]
change = change[change != np.inf]
return np.mean(change )<load_from_csv>
all_data = all_data.drop(item_mean_features, axis=1)
 all_data = all_data.drop(shop_mean_features, axis=1)
 all_data = all_data.drop(cat_mean_features, axis=1)
 all_data = all_data.drop(all_data[all_data["date_block_num"]<3].index )
class FeatureGenerator(object):
def __init__(self, dtype, n_jobs=1, chunk_size=None):
self.chunk_size = chunk_size
self.dtype = dtype
self.filename = None
self.n_jobs = n_jobs
self.test_files = []
if self.dtype == 'train':
self.filename = '.. /input/train.csv'
self.total_data = int(629145481 / self.chunk_size)
 else:
submission = pd.read_csv('.. /input/sample_submission.csv')
 for seg_id in submission.seg_id.values:
self.test_files.append(( seg_id, '.. /input/test/' + seg_id + '.csv'))
self.total_data = int(len(submission))
def read_chunks(self):
if self.dtype == 'train':
iter_df = pd.read_csv(self.filename, iterator=True, chunksize=self.chunk_size,
dtype={'acoustic_data': np.float64, 'time_to_failure': np.float64})
 for counter, df in enumerate(iter_df):
x = np.sign(df.acoustic_data.values)*np.log1p(np.abs(df.acoustic_data.values))
y = df.time_to_failure.values[-1]
seg_id = 'train_' + str(counter)
 del df
yield seg_id, x, y
else:
for seg_id, f in self.test_files:
df = pd.read_csv(f, dtype={'acoustic_data': np.float64})
 x = df.acoustic_data.values[-self.chunk_size:]
x = np.sign(x)*np.log1p(np.abs(x))
del df
yield seg_id, x, -999
def get_features(self, x, y, seg_id):

 x = pd.Series(x)
 zc = np.fft.fft(x)
 realFFT = pd.Series(np.real(zc))
imagFFT = pd.Series(np.imag(zc))
main_dict = self.features(x, y, seg_id)
 r_dict = self.features(realFFT, y, seg_id)
 i_dict = self.features(imagFFT, y, seg_id)
 for k, v in r_dict.items() :
if k in ['classic_sta_lta2_mean',
'exp_Moving_std_3000_mean',
'classic_sta_lta1_mean']:
main_dict[f'fftr_{k}'] = v
for k, v in i_dict.items() :
if k in ['classic_sta_lta2_mean']:
main_dict[f'ffti_{k}'] = v
return main_dict
def features(self, x, y, seg_id):
feature_dict = dict()
feature_dict['target'] = y
feature_dict['seg_id'] = seg_id
for p in [50]:
feature_dict[f'abs_percentile_{p}'] = np.percentile(np.abs(x), p)
 for autocorr_lag in [5]:
feature_dict[f'autocorrelation_{autocorr_lag}'] = feature_calculators.autocorrelation(x, autocorr_lag)
 for p in [95,99]:
feature_dict[f'binned_entropy_{p}'] = feature_calculators.binned_entropy(x, p)
 for slice_length, direction in product([50000], ['first']):
if direction == 'first':
feature_dict[f'mean_change_rate_{direction}_{slice_length}'] = calc_change_rate(x[:slice_length])
 elif direction == 'last':
feature_dict[f'mean_change_rate_{direction}_{slice_length}'] = calc_change_rate(x[-slice_length:])
 for peak in [10]:
feature_dict[f'num_peaks_{peak}'] = feature_calculators.number_peaks(x, peak)
 for p in [20]:
feature_dict[f'percentile_{p}'] = np.percentile(x, p)
 x_roll_std = x.rolling(1000 ).std().dropna().values
feature_dict[f'percentile_roll_std_30_window_1000'] = np.percentile(x_roll_std, 30)
 x_roll_std = x.rolling(500 ).std().dropna().values
feature_dict[f'percentile_roll_std_75_window_500'] = np.percentile(x_roll_std, 75)
 feature_dict[f'percentile_roll_std_80_window_500'] = np.percentile(x_roll_std, 80)
 ewma = pd.Series.ewm
feature_dict[f'exp_Moving_std_3000_mean'] =(ewma(x, span=3000 ).std(skipna=True)).mean(skipna=True)
 feature_dict['classic_sta_lta1_mean'] = classic_sta_lta(x, 500, 10000 ).mean()
feature_dict['classic_sta_lta2_mean'] = classic_sta_lta(x, 5000, 100000 ).mean()
feature_dict['skew'] = x.skew()
return feature_dict
def generate(self):
feature_list = []
res = Parallel(n_jobs=self.n_jobs,
backend='threading' )(delayed(self.get_features )(x, y, s)
 for s, x, y in tqdm_notebook(self.read_chunks() , total=self.total_data))
for r in res:
feature_list.append(r)
 return pd.DataFrame(feature_list )<features_selection>
del X_test_temp
gc.collect()
training_fg = FeatureGenerator(dtype='train', n_jobs=4, chunk_size=150000)
 training_data = training_fg.generate()
training_data.columns<define_variables>
all_data["item_cnt_month_3lag_mean"] = all_data[["item_cnt_month_lag1", "item_cnt_month_lag2", "item_cnt_month_lag3"]].mean(axis=1 )
testing_fg = FeatureGenerator(dtype='test', n_jobs=4, chunk_size=150000)
 test_data = testing_fg.generate()<prepare_x_and_y>
all_data[lag_feats_to_clip + ["item_cnt_month_3lag_mean",
"item_cnt_month"]] = all_data[lag_feats_to_clip + ["item_cnt_month_3lag_mean",
"item_cnt_month"]].clip(0,20 )
X = training_data.drop(['target', 'seg_id'], axis=1)
 X_test = test_data.drop(['target', 'seg_id'], axis=1)
 test_segs = test_data.seg_id
y = training_data.target<normalization>
all_data["lag_grad_1"] = all_data["item_cnt_month_lag1"] / all_data["item_cnt_month_lag2"]
all_data["lag_grad_1"] = all_data["lag_grad_1"].replace([np.inf, -np.inf], np.nan ).fillna(0)
 all_data["lag_grad_2"] = all_data["item_cnt_month_lag2"] / all_data["item_cnt_month_lag3"]
all_data["lag_grad_2"] = all_data["lag_grad_2"].replace([np.inf, -np.inf], np.nan ).fillna(0 )
ss = StandardScaler()
feats = X_test.columns
ss.fit(pd.concat([X[feats],X_test[feats]]))
X[feats] = ss.transform(X[feats])
 X_test[feats] = ss.transform(X_test[feats])
 X.insert(0,'seg_id',training_data.seg_id.values)
 X['time_to_failure'] = y.values
X_test.insert(0,'seg_id',test_segs.values)
<compute_test_metric>
all_data["new_items"] = all_data["first_sale_date"] == all_data["date_block_num"]
def GPI(data):
return(5.683668 +
1.0*np.tanh(((((data["abs_percentile_50"])-(((data["percentile_roll_std_75_window_500"])* 2.0)))) -(((((((((((data["percentile_roll_std_75_window_500"])+(data["num_peaks_10"])))* 2.0)) +(data["abs_percentile_50"])))* 2.0)) * 2.0)))) +
1.0*np.tanh(((-1.0)-(((((data["percentile_roll_std_30_window_1000"])+(((( data["binned_entropy_95"])+(data["num_peaks_10"])) /2.0)))) +(data["percentile_roll_std_75_window_500"])))))+
1.0*np.tanh(((((((((data["abs_percentile_50"])+(data["binned_entropy_99"])))*(((( data["percentile_roll_std_75_window_500"])+(data["num_peaks_10"])) /2.0)))) *(( -1.0*(((( data["percentile_roll_std_75_window_500"])+(data["percentile_roll_std_30_window_1000"])))))))) * 2.0)) +
1.0*np.tanh(((( -1.0*(((((( data["fftr_exp_Moving_std_3000_mean"])*(((1.0)+(((data["percentile_roll_std_30_window_1000"])* 2.0)))))) / 2.0)))))*(((( data["num_peaks_10"])+(data["abs_percentile_50"])) /2.0)))) +
1.0*np.tanh(((((data["exp_Moving_std_3000_mean"])-(data["percentile_roll_std_75_window_500"])))+(((data["num_peaks_10"])*(((data["fftr_classic_sta_lta1_mean"])-(data["fftr_exp_Moving_std_3000_mean"]))))))))
def GPII(data):
return(5.683668 +
1.0*np.tanh(((((((((( -1.0*(((( data["num_peaks_10"])+(data["percentile_roll_std_75_window_500"])))))) * 2.0)) -(((data["num_peaks_10"])+(data["abs_percentile_50"])))))* 2.0)) * 2.0)) +
1.0*np.tanh(((((((( data["exp_Moving_std_3000_mean"])+(-2.0)))+(((((((data["percentile_roll_std_80_window_500"])+(data["autocorrelation_5"])))/ 2.0)) / 2.0)))/2.0)) -(((((data["percentile_roll_std_75_window_500"])* 2.0)) * 2.0)))) +
1.0*np.tanh(((( -1.0*(( data["fftr_exp_Moving_std_3000_mean"])))) *(((((data["num_peaks_10"])+(data["abs_percentile_50"])))*(((((data["percentile_roll_std_30_window_1000"])* 2.0)) +(((1.0)+(data["percentile_roll_std_30_window_1000"])))))))))+
1.0*np.tanh(((data["exp_Moving_std_3000_mean"])-(((data["percentile_roll_std_75_window_500"])+(((((((( data["abs_percentile_50"])+(data["percentile_roll_std_75_window_500"])) /2.0)) *(((( data["abs_percentile_50"])+(data["percentile_roll_std_75_window_500"])) /2.0)))) *(data["fftr_exp_Moving_std_3000_mean"])))))))+
1.0*np.tanh(((((((((( data["percentile_roll_std_80_window_500"])-(data["fftr_exp_Moving_std_3000_mean"])))+(((data["fftr_classic_sta_lta1_mean"])-(data["fftr_classic_sta_lta2_mean"])))) /2.0)) +(((data["fftr_classic_sta_lta1_mean"])-(data["fftr_exp_Moving_std_3000_mean"])))))*(data["num_peaks_10"]))))<compute_test_metric>
all_data["time_since_first_sale"] = all_data["date_block_num"] - all_data["first_sale_date"]
all_data.drop("first_sale_date", inplace=True, axis=1 )
mean_absolute_error(X.time_to_failure,GPI(X))<compute_test_metric>
all_data["month"] = all_data["date_block_num"] % 12
mean_absolute_error(X.time_to_failure,GPII(X))<save_to_csv>
all_data.drop(["item_cnt"], axis=1, inplace=True )
X_test['time_to_failure'] =(GPI(X_test)+GPII(X_test)) /2.
 X_test[['seg_id','time_to_failure']].to_csv('submission.csv',index=False )<set_options>
all_data["category_id"] = all_data["category_id"].astype("int8" )
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
 print(os.listdir(".. /input"))
print(os.listdir(".. /input/gplearnmodels")) 
<load_from_csv>
X_train = all_data[all_data["date_block_num"]<33]
y_train = X_train["item_cnt_month"]
X_train = X_train.drop("item_cnt_month", axis=1)
 X_val = all_data[all_data["date_block_num"] == 33]
y_val = X_val["item_cnt_month"]
X_val = X_val.drop("item_cnt_month", axis=1)
 X_test = all_data[all_data["date_block_num"]==34]
X_test = X_test.drop("item_cnt_month", axis=1)
 del all_data
gc.collect()
df=pd.read_csv('.. /input/andrewstsfreshv19/ExtFeaturestrain.csv')
 test=pd.read_csv('.. /input/andrewstsfreshv19/ExtFeaturestest.csv')
 print(df.shape)
 X=df[df.columns.difference(['time_to_failure'])]
y=df['time_to_failure']
y_tr=y
scaler=MinMaxScaler(feature_range=(0, 1))
scaled=scaler.fit_transform(X.append(test))
train=pd.DataFrame(scaled[:X.shape[0]], columns=X.columns.values)
 test=pd.DataFrame(scaled[-test.shape[0]:], columns=X.columns.values)
 trainX, testX, trainy, testy = train_test_split(train, y, test_size=0.0,shuffle=True,random_state=42)
 X_tr_scaled = trainX
X_test_scaled = test<choose_model_class>
def preds(model, test, name):

 prediction = model.predict(test)
 df_sub = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sample_submission.csv")
 df_sub["item_cnt_month"] = prediction.clip(0,20)
 df_sub.to_csv(f"{name}.csv", index=False)
 print("Complete." )
est_gp = SymbolicRegressor(population_size=10000,
tournament_size=50,
generations=1000,
stopping_criteria=0.0,
p_crossover=0.9, p_subtree_mutation=0.0001, p_hoist_mutation=0.0001, p_point_mutation=0.0001,
max_samples=1.0, verbose=1,
function_set =('tan', 'add', 'sub', 'mul', 'div','max', 'min'),
metric = 'mean absolute error', warm_start=True,
n_jobs = -1, parsimony_coefficient=0.00003, random_state=11 )<load_pretrained>
params = {'metric': 'rmse',
'num_leaves': 255,
'learning_rate': 0.005,
'feature_fraction': 0.75,
'bagging_fraction': 0.75,
'bagging_freq': 5,
'force_col_wise' : True,
'random_state': 10,
'num_rounds':1500,
'early_stopping':150}
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_val = lgb.Dataset(X_val, y_val)
 model = lgb.train(params=params, train_set=lgb_train, valid_sets=(lgb_train, lgb_val), verbose_eval=50 )
with open('.. /input/gplearnmodels/gp_model.pkl', 'rb')as f:
est_gp = pickle.load(f )<train_model>
preds(model, X_test, "lgb_model" )
n=850
class TimeoutException(Exception):
pass
def timeout_handler(signum, frame):
raise TimeoutException
signal.signal(signal.SIGALRM, timeout_handler)
 signal.alarm(n)
 try:
est_gp.fit(train, y)
 except TimeoutException:
print('function terminated' )<train_model>
del lgb_train, lgb_val
gc.collect()
<save_to_csv><EOS> 
xgb_model = XGBRegressor(
 max_depth=8,
n_estimators=100,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.1,
seed=42)
 xgb_model.fit(X_train, y_train, eval_metric="rmse", eval_set=[(X_train, y_train),(X_val, y_val)])
 preds(xgb_model, X_test, f"xgb_{i}")
 plot_features(xgb_model,(10, 14))