Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<load_from_csv>
!pip install textdistance
data = {k.split('/')[-1][:-4]:k for k in glob.glob('/kaggle/input/**/**.csv')}
train = pd.read_csv(data['jigsaw-toxic-comment-train'], usecols=['id', 'comment_text', 'toxic'])
 val = pd.read_csv(data['validation'], usecols=['comment_text', 'toxic'])
 test = pd.read_csv(data['test'], usecols=['id', 'content'])
 test.columns = ['id', 'comment_text']
test['toxic'] = 0.5
sub2 = pd.read_csv('.. /input/ensemble/submission.csv' )<feature_engineering>
%matplotlib inline
%%time
def f_experience(c, s):
it = {'memory':10,
'influence':0.5,
'inference':0.5,
'interest':0.9,
'sentiment':1e-10,
'harmony':0.5}
exp = {}
for i in range(len(c)) :
words = set([w for w in str(c[i] ).lower().split(' ')])
 for w in words:
try:
exp[w]['influence'] = exp[w]['influence'][1:] + [s[i]]
exp[w]['inference'] += 1
exp[w]['interest'] = exp[w]['interest'][1:] + [(exp[w]['interest'][it['memory']-1] +(s[i] * it['interest'])) /2]
exp[w]['sentiment'] += s[i]
except:
m = [0.for m_ in range(it['memory'])]
exp[w] = {}
exp[w]['influence'] = m[1:] + [s[i]]
exp[w]['inference'] = 1
exp[w]['interest'] = m[1:] + [s[i] * it['interest'] / 2]
exp[w]['sentiment'] = s[i]
for w in exp:
exp[w]['sentiment'] /= exp[w]['inference'] + it['sentiment']
exp[w]['inference'] /= len(c)* it['inference']
return exp
exp = f_experience(train['comment_text'].values, train['toxic'].values )<feature_engineering>
def downcast(df):
df_dtypes = df.dtypes
new_dtypes = []
for idx, col_dtype in enumerate(df_dtypes):
if col_dtype == "float64":
new_dtypes.append(np.float16)
 elif col_dtype in ["int64", "int32", "int16"]:
if df[df.columns[idx]].min() >= -128 and df[df.columns[idx]].max() <= 127:
new_dtypes.append(np.int8)
 elif df[df.columns[idx]].min() >= -32768 and df[df.columns[idx]].max() <= 32767:
new_dtypes.append(np.int16)
 elif df[df.columns[idx]].min() >= -2147483648 and df[df.columns[idx]].max() <= 2147483647:
new_dtypes.append(np.int32)
 else:
new_dtypes.append(np.int64)
 else:
new_dtypes.append(col_dtype)
 return df.astype(dict(zip(df.columns,new_dtypes)) )
%%time
def features(df):
df['len'] = df['comment_text'].map(len)
 df['wlen'] = df['comment_text'].map(lambda x: len(str(x ).split(' ')))
 df['influence_sum'] = df['comment_text'].map(lambda x: np.sum([np.mean(exp[w]['influence'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['influence_mean'] = df['comment_text'].map(lambda x: np.mean([np.mean(exp[w]['influence'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['inference_sum'] = df['comment_text'].map(lambda x: np.sum([exp[w]['inference'] if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['inference_mean'] = df['comment_text'].map(lambda x: np.mean([exp[w]['inference'] if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['interest_sum'] = df['comment_text'].map(lambda x: np.sum([np.mean(exp[w]['interest'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['interest_mean'] = df['comment_text'].map(lambda x: np.mean([np.mean(exp[w]['interest'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['sentiment_sum'] = df['comment_text'].map(lambda x: np.sum([exp[w]['sentiment'] if w in exp else 0.5 for w in str(x ).lower().split(' ')]))
df['sentiment_mean'] = df['comment_text'].map(lambda x: np.mean([exp[w]['sentiment'] if w in exp else 0.5 for w in str(x ).lower().split(' ')]))
return df
val = features(val)
 test= features(test )<train_on_grid>
train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 item_categories = pd.read_pickle('.. /input/translated/item_categories.pkl')
 shops = pd.read_pickle('.. /input/translated/shops.pkl' )
col = [c for c in val if c not in ['id', 'comment_text', 'toxic']]
x1, x2, y1, y2 = model_selection.train_test_split(val[col], val['toxic'], test_size=0.3, random_state=20)
 model = ensemble.ExtraTreesClassifier(n_estimators=1000, max_depth=7, n_jobs=-1, random_state=20)
 model.fit(x1, y1)
 print(metrics.roc_auc_score(y2, model.predict_proba(x2)[:,1].clip(0.,1.)))
model.fit(val[col], val['toxic'])
 test['toxic'] = model.predict_proba(test[col])[:,1].clip(0.,1.)
sub1 = test[['id', 'toxic']]<save_to_csv>
pd.options.display.precision = 6
pd.options.display.float_format = '{:.2f}'.format
sub1.rename(columns={'toxic':'toxic1'}, inplace=True)
 sub2.rename(columns={'toxic':'toxic2'}, inplace=True)
 sub3 = pd.merge(sub1, sub2, how='left', on='id')
 sub3['toxic'] =(sub3['toxic1'] * 0.1)+(sub3['toxic2'] * 0.9)
 sub3['toxic'] =(sub3['toxic2'] * 0.51)+(sub3['toxic'] * 0.49)
 sub3[['id', 'toxic']].to_csv('submission.csv', index=False )<prepare_output>
print(train.duplicated().count())
 train = train[~train.duplicated() ]
train.info()
test = pd.DataFrame(['Howling with Wolf on Lügenpresse'], columns=['comment_text'])
 test['id'] = test.index
test= features(test)
 test['toxic'] = model.predict_proba(test[col])[:,1].clip(0.,1.)
test[['id', 'comment_text', 'toxic']].head()<set_options>
train["item_id"].value_counts().sort_values(ascending=False)[:10]
warnings.filterwarnings("ignore")
 tqdm.pandas()
CountVectorizer,\
HashingVectorizer
stopword=set(STOPWORDS)
 lem = WordNetLemmatizer()
tokenizer=TweetTokenizer()
np.random.seed(0)
<choose_model_class>
train["shop_id"].value_counts(normalize=True ).sort_values(ascending=False)[:10]
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print(strategy.num_replicas_in_sync)
 BATCH_SIZE = 16 * strategy.num_replicas_in_sync<string_transform>
train["item_id"].value_counts().sort_values(ascending=False)[:10]
MODEL = 'jplu/tf-xlm-roberta-base'
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
 text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
tokenized_text = tokenizer.tokenize(text )<define_variables>
items.loc[items["item_id"]==20949]
masked_index = 8
tokenized_text[masked_index] = '[MASK]'
assert tokenized_text == ['[CLS]', 'who', 'was', 'jim', 'henson', '?', '[SEP]', 'jim', '[MASK]', 'was', 'a', 'puppet', '
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
 segments_ids = [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]
tokens_tensor = torch.tensor([indexed_tokens])
 segments_tensors = torch.tensor([segments_ids] )<predict_on_test>
item_categories.loc[item_categories["item_category_id"]==71]
model = BertForMaskedLM.from_pretrained('bert-base-uncased')
 model.eval()
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print(device)
 tokens_tensor = tokens_tensor.to(device)
 segments_tensors = segments_tensors.to(device)
 model.to(device)
 with torch.no_grad() :
outputs = model(tokens_tensor, token_type_ids=segments_tensors)
 predictions = outputs[0]
predicted_index = torch.argmax(predictions[0, masked_index] ).item()
predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]
assert predicted_token == 'henson'
print('Predicted token is:',predicted_token )<import_modules>
train["item_price"].sort_values(ascending=False)[:10]
import pandas as pd<import_modules>
train[train['item_price'] == 307980]
import pandas as pd<load_from_csv>
items.loc[items["item_id"]==6066]
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 train3 = pd.read_csv('.. /input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')
 train4 = pd.read_csv('.. /input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 toxic = len(train2[['comment_text', 'toxic']].query('toxic==1'))
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train3[['comment_text', 'toxic']].query('toxic==0'),
train3[['comment_text', 'toxic']].query('toxic==1'),
train4[['comment_text', 'toxic']].query('toxic==0'),
train4[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=(toxic+(toxic//3)) , random_state=101)
 ])
 test_data = test
train_data = train
maxlen = 96<load_from_csv>
item_categories.loc[item_categories["item_category_id"]==75]
df_valid_en = pd.read_csv('.. /input/val-en-df/validation_en.csv')
 df_valid_en = df_valid_en.drop(columns=['id', 'comment_text','lang'])
 df_valid_en = df_valid_en.rename(columns={"comment_text_en": "comment_text"})
 columns_titles = ["comment_text","toxic"]
df_valid_en=df_valid_en.reindex(columns=columns_titles)

print(df_valid_en.head(3))
train_data = pd.concat([train_data , df_valid_en], axis=0 ).reset_index(drop=True)
 train_data = train_data.sample(frac=1 ).reset_index(drop=True)
 print(len(train2[['comment_text', 'toxic']].query('toxic==1')))
 print(len(train_data))
train_data.head() 
<feature_engineering>
test[test["item_id"]==6066].empty
val = valid
train = train_data
def clean(text):
text = text.fillna("fillna" ).str.lower()
text = text.map(lambda x: re.sub('\
',' ',str(x)))
 text = text.map(lambda x: re.sub("\[\[User.*",'',str(x)))
 text = text.map(lambda x: re.sub("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}",'',str(x)))
 text = text.map(lambda x: re.sub("\(http://.*?\s\(http://.*\)",'',str(x)))
 return text
val["comment_text"] = clean(val["comment_text"])
 test_data["content"] = clean(test_data["content"])
 train["comment_text"] = clean(train["comment_text"] )<define_variables>
train[train['item_price'] < 0]
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '\xa0', '\t',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '\u3000', '\u202f',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '«',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"couldnt" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"doesnt" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"havent" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"shouldnt" : "should not",
"that's" : "that is",
"thats" : "that is",
"there's" : "there is",
"theres" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"theyre": "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def clean_text(x):
x = str(x ).replace("
","")
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x<string_transform>
items.loc[items["item_id"]==2978]
tokenizer = TreebankWordTokenizer()
def handle_contractions(x):
x = tokenizer.tokenize(x)
 return x
def fix_quote(x):
x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
def replace_typical_misspell(text):
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def clean_data(df, columns: list):
for col in columns:
df[col] = df[col].apply(lambda x: clean_text(x.lower()))
df[col] = df[col].apply(lambda x: replace_typical_misspell(x))
df[col] = df[col].apply(lambda x: handle_contractions(x))
df[col] = df[col].apply(lambda x: fix_quote(x))
return df 
<categorify>
item_categories.loc[item_categories["item_category_id"]==31]
%%time
input_columns = [
'comment_text'
]

 train = clean_data(train, input_columns)
 val = clean_data(val, input_columns)
<drop_column>
train = train[train["item_id"] != 2978]
%%time
input_columns = [
'content'
]
test_data = clean_data(test_data, input_columns)
 del tokenizer<compute_train_metric>
train["item_cnt_day"].sort_values(ascending=False)[:10]
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))<categorify>
train[train["item_cnt_day"] > 999]
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<load_pretrained>
items.loc[items["item_id"]==11373]
MODEL = 'jplu/tf-xlm-roberta-large'
tokenizer = AutoTokenizer.from_pretrained(MODEL)
 save_path = '/kaggle/working/xlmr_large/'
if not os.path.exists(save_path):
os.makedirs(save_path)
 tokenizer.save_pretrained(save_path)
<prepare_x_and_y>
item_categories.loc[item_categories["item_category_id"]==9]
%%time
x_train = regular_encode(train.comment_text.astype(str),
tokenizer, maxlen=maxlen)
 x_valid = regular_encode(val.comment_text.astype(str ).values,
tokenizer, maxlen=maxlen)
 x_test = regular_encode(test_data.content.astype(str ).values,
tokenizer, maxlen=maxlen)
 y_valid = val.toxic.values
y_train = train.toxic.values<categorify>
train[train["item_id"]==11373]["item_cnt_day"].sort_values(ascending=False)[:10]
%%time
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<compute_train_metric>
train = train[train["item_cnt_day"] < 1000]
def focal_loss(gamma=2., alpha=.2):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) - K.mean(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return focal_loss_fixed<choose_model_class>
print(test[["shop_id", "item_id"]].duplicated().sum() )
def build_model(transformer, loss='binary_crossentropy', max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
x = tf.keras.layers.Dropout(0.2 )(cls_token)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=3e-5), loss=loss, metrics=[tf.keras.metrics.AUC() ])
 return model<load_pretrained>
test["shop_id"].value_counts().sort_values() [:10]
%%time
with strategy.scope() :
transformer_layer = transformers.TFXLMRobertaModel.from_pretrained(MODEL)
 model = build_model(transformer_layer,loss='binary_crossentropy', max_len=maxlen)
 model.summary()<compute_train_metric>
test["shop_id"].value_counts().sort_values() [-5:]
def callback() :
cb = []
reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss',
factor=0.3, patience=2,
verbose=1, mode='auto',
epsilon=0.0001, cooldown=1, min_lr=0.000001)
 cb.append(reduceLROnPlat)
 log = CSVLogger('log.csv')
 cb.append(log)
 RocAuc = RocAucEvaluation(validation_data=(x_valid, y_valid), interval=1)
 cb.append(RocAuc)
 return cb<choose_model_class>
test["item_id"].value_counts().sort_values() [:10]
def build_lrfn(lr_start=0.000001, lr_max=0.000002,
lr_min=0.0000001, lr_rampup_epochs=7,
lr_sustain_epochs=0, lr_exp_decay=.87):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn<define_variables>
test["item_id"].value_counts().sort_values() [-10:]
model_path = 'jigsawMultilingual.hdf5'
model_path1 = '/kaggle/working/jigsawMultilingual.hdf5'<choose_model_class>
mat = np.zeros(( shops.shape[0], shops.shape[0]))
checkpoint = ModelCheckpoint(model_path, monitor='val_accuracy', mode='max', save_best_only=True)
 es = EarlyStopping(monitor='val_accuracy', mode='max', patience=2,
restore_best_weights=True, verbose=1)
 lr_callback = LearningRateScheduler(lrfn, verbose=1)
 callback_list = [checkpoint, lr_callback]<train_model>
for row in range(shops.shape[0]):
for col in range(shops.shape[0]):
mat[row, col] = textdistance.levenshtein.normalized_similarity(shops["shop_name"][row], shops["shop_name"][col])
 mat = np.where(mat == 1 , 0, mat )
%%time
N_STEPS = x_train.shape[0] // BATCH_SIZE
EPOCHS = 2
train_history = model.fit(
 train_dataset,
steps_per_epoch=N_STEPS,
validation_data=valid_dataset,
callbacks=callback_list,
epochs=EPOCHS
)<load_pretrained>
shop_name_duplicated = pd.DataFrame(np.transpose(np.nonzero(np.where(mat > 0.7, mat, 0))))
shop_name_duplicated = shop_name_duplicated[~shop_name_duplicated.apply(frozenset, axis=1 ).duplicated() ].reset_index(drop=True)
 print(shop_name_duplicated )
if os.path.exists(model_path1):
model.load_weights(model_path1 )<train_model>
for index, row in shop_name_duplicated.iterrows() :
print("shop_id:", row[0], "shop_name:", shops.loc[shops["shop_id"] == row[0], "shop_name"].values[0])
 print("shop_id:", row[1], "shop_name:", shops.loc[shops["shop_id"] == row[1], "shop_name"].values[0])
 print()
%%time
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
callbacks=callback_list,
epochs= EPOCHS
)<load_pretrained>
test[test.isin({"shop_id":[0,57,1,58,10,11,23,24]})["shop_id"]]["shop_id"].drop_duplicates()
model.save_weights('model.h5' )<load_pretrained>
train.loc[train["shop_id"]== 0, "shop_id"] = 57
test.loc[test["shop_id"]== 0, "shop_id"] = 57
train.loc[train["shop_id"]== 1, "shop_id"] = 58
test.loc[test["shop_id"]== 1, "shop_id"] = 58
train.loc[train["shop_id"]== 23, "shop_id"] = 24
test.loc[test["shop_id"]== 23, "shop_id"] = 24
train.loc[train["shop_id"]== 11, "shop_id"] = 10
test.loc[test["shop_id"]== 11, "shop_id"] = 10
if os.path.exists(model_path1):
model.load_weights(model_path1 )<drop_column>
print("shop_id:", "train:" , len(train["shop_id"].unique()), "test:", len(test["shop_id"].unique()))
print("item_id:", "train:" , len(train["item_id"].unique()), "test:", len(test["item_id"].unique()))
log_dir = "/kaggle/working/log.csv"
if os.path.exists(log_dir):
os.remove(log_dir )<save_to_csv>
train_comb = train[["shop_id", "item_id"]].drop_duplicates()
print(train_comb )
sub = pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/' + 'sample_submission.csv')
 sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<import_modules>
test_comb = test[["shop_id", "item_id"]].drop_duplicates()
print(test_comb )
from tensorflow.keras.layers import Dense, Flatten, Dropout, Conv2D, MaxPool2D, BatchNormalization, AvgPool2D
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.utils import to_categorical
from sklearn.metrics import confusion_matrix
from tensorflow.random import set_seed
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
unseen_shops = test_comb.shop_id[~test_comb["shop_id"].isin(train_comb["shop_id"])].drop_duplicates()
unseen_items = test_comb.item_id[~test_comb["item_id"].isin(train_comb["item_id"])].drop_duplicates()
print("In the test set:", unseen_shops.count() , "unseen shops","and", unseen_items.count() , "unseen items" )
df_train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 df_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )<define_variables>
len(test["item_id"].unique())- unseen_items.count()
np.random.seed(42)
 set_seed(42 )<save_to_csv>
unseen_combs = []
seen_others = []
for shop in test_comb["shop_id"].unique() :
test_items = test_comb.loc[test_comb["shop_id"] == shop, ["item_id"]].values.reshape(-1)
 train_items = train_comb.loc[train_comb["shop_id"] == shop, ["item_id"]].values.reshape(-1)
 train_items_others = train_comb.loc[train_comb["shop_id"] != shop, ["item_id"]].values.reshape(-1)
 for item in test_items:
if item not in train_items:
unseen_combs.append(( shop, item))
def doSubmission(y_pred):
test_Id = np.arange(1, y_pred.size+1, dtype=np.int)
 pred_dict = {"ImageId": test_Id, "Label": y_pred}
df = pd.DataFrame(pred_dict)
 df.to_csv("sample_submission.csv", index=False, index_label=False )<data_type_conversions>
unseen_comb_items = unseen_combs_df["item_id"].unique()
len(unseen_comb_items )
y = df_train.label.to_numpy()
X = df_train.drop(columns=["label"] ).to_numpy()
X = X / 255.0<train_model>
214200 - 102697
X_totrain = X.reshape(X.shape[0], 28, 28, 1 )<split>
last_months_shops_sales = train.query('date_block_num > 27' ).groupby("shop_id")["item_cnt_day"].sum()
print(last_months_shops_sales > 0 )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, stratify=y)
 X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
 X_test = X_test.reshape(X_test.shape[0], 28, 28, 1 )<categorify>
last_months_sales_shops = last_months_shops_sales.index.tolist()
y_cat = to_categorical(y, 10)
 y_train_cat = to_categorical(y_train, 10)
 y_test_cat = to_categorical(y_test, 10 )<data_type_conversions>
last_months_items_sales = train.query('date_block_num > 27' ).groupby("item_id")["item_cnt_day"].sum()
last_months_items_sales = last_months_items_sales[last_months_items_sales > 0]
print(last_months_items_sales )
test = df_test.to_numpy(np.float64)
 test = test.reshape(test.shape[0], 28, 28, 1)
 test /= 255.0<choose_model_class>
last_months_sales_items = last_months_items_sales.index.tolist()
def convNeuralNetwork() :
cnn = Sequential()
cnn.add(Conv2D(filters=6, kernel_size=(5, 5), strides=(1, 1),
input_shape=(28, 28, 1), activation="tanh", padding="same"))
cnn.add(BatchNormalization())
 cnn.add(AvgPool2D(pool_size=(2, 2), strides=(1, 1), padding="same"))
cnn.add(Dropout(0.2))
cnn.add(Conv2D(16, kernel_size=(5, 5), strides=(1, 1),
activation="tanh", padding="same"))
cnn.add(BatchNormalization())
 cnn.add(AvgPool2D(pool_size=(2, 2), strides=(2, 2), padding="same"))
cnn.add(Dropout(0.2))
cnn.add(Conv2D(120, kernel_size=(5, 5), strides=(1, 1),
activation="tanh", padding="same"))
cnn.add(AvgPool2D(pool_size=(2, 2), strides=(2, 2), padding="same"))
cnn.add(Dropout(0.2))
cnn.add(Flatten())
 cnn.add(Dense(units=84, activation="tanh"))
cnn.add(Dropout(0.2))
cnn.add(Dense(units=10, activation="softmax"))
cnn.compile(optimizer="SGD", loss="categorical_crossentropy",
metrics=["accuracy"])
 return cnn<train_model>
len(set(no_sales_items)& set(unseen_items))
early_stopping = EarlyStopping(monitor="val_loss", patience=4, verbose=1,
restore_best_weights=True)
 lr_reducer = ReduceLROnPlateau(monitor="val_loss", factor=0.1, patience=3,
verbose=1)
 cnn = convNeuralNetwork()
cnn_hist = cnn.fit(X_train, y_train_cat, validation_data=(X_test, y_test_cat),
epochs=100, batch_size=16, callbacks=[early_stopping, lr_reducer] )<train_on_grid>
outdated_items = list(set(no_sales_items)- set(unseen_items))
len(outdated_items )
early_stopping = EarlyStopping(monitor="loss", patience=4, restore_best_weights=True, verbose=1)
 model_checkpoint = ModelCheckpoint(filepath="./", monitor="loss", verbose=1,
save_best_only=True, save_weights_only=True)
 lr_reducer = ReduceLROnPlateau(monitor="loss", factor=0.1, patience=3,
verbose=1)
 model= convNeuralNetwork()
model_hist = model.fit(X_totrain, y_cat, epochs=100, batch_size=16,
callbacks=[early_stopping, lr_reducer, model_checkpoint] )<predict_on_test>
no_split = item_categories.item_category_name.str.split(" ", expand=True, n = 2)
 no_split = no_split.fillna("" ).apply(lambda x: x.replace("-", ""))
no_split[0] = no_split[0] + " " + no_split[1]
item_category_df = no_split.drop(columns = 1)
 item_category_df.columns = ["big_category_name", "sub_category_name"]
big_category_name = item_category_df.big_category_name.str.strip()
sub_category_name = item_category_df.sub_category_name.apply(lambda x: x.replace("-", "")).str.strip()
item_category_df["big_category_name"] = big_category_name
item_category_df["sub_category_name"] = sub_category_name
item_category_df["big_category_id"] = item_category_df.big_category_name.astype("category" ).cat.codes.to_frame(name = "big_category_id")
 item_category_df["sub_category_id"] = item_category_df.sub_category_name.astype("category" ).cat.codes.to_frame(name = "sub_category_id")
 item_categories = item_categories.join(item_category_df)
 print(item_categories)
 item_categories = downcast(item_categories)
 del no_split
del item_category_df
y_pred = model.predict(test ).argmax(1)
 doSubmission(y_pred )<load_from_csv>
split = shops.shop_name
split = split.apply(lambda x:x.replace("!", ""))
without_split = split.str.split(expand=True)
 to_correct = without_split.iloc[[34, 35, 42, 43, 46]].drop(columns = [3,4,5,6,7])
 to_correct[0] = to_correct[0] + " " + to_correct[1]
to_correct[1] = to_correct[2]
to_correct.drop(columns = 2, inplace=True)
 without_split_new = without_split.drop(index= [34, 35, 42, 43, 46] ).drop(columns = [2,3,4,5,6,7])
 without_split_new = to_correct.join(without_split_new, how="outer", rsuffix="_" ).fillna("")
 without_split_new["0"] = without_split_new["0"] + without_split_new["0_"]
without_split_new["1"] = without_split_new["1"] + without_split_new["1_"]
without_split_new.drop(columns = ["0_","1_"], inplace=True)
 without_split_new.columns = ["city_name", "shop_category_name"]
without_split_new.city_name = without_split_new.city_name.str.strip()
without_split_new.shop_category_name = without_split_new.shop_category_name.str.strip()
city_name_ids = without_split_new.city_name.astype("category" ).cat.codes.to_frame(name = "city_id")
 shop_category_ids = without_split_new.shop_category_name.astype("category" ).cat.codes.to_frame(name = "shop_category_id")
 shop_df = without_split_new.join(city_name_ids ).join(shop_category_ids)
 shops = shops.join(shop_df)
 print(shops)
 shops = downcast(shops)
 del to_correct
del shop_df
mnist_test = pd.read_csv(".. /input/mnist-in-csv/mnist_test.csv")
 mnist_train = pd.read_csv(".. /input/mnist-in-csv/mnist_train.csv" )<load_from_csv>
print("train")
 print(train.isna().sum())
 print("test")
 print(test.isna().sum())
 print("item_categories")
 print(item_categories.isna().sum())
 print("items")
 print(items.isna().sum())
 print("shops")
 print(shops.isna().sum())
 print("train")
 print(train.isnull().sum())
 print("test")
 print(test.isnull().sum())
 print("item_categories")
 print(item_categories.isnull().sum())
 print("items")
 print(items.isnull().sum())
 print("shops")
 print(shops.isnull().sum() )
sample_submission = pd.read_csv(".. /input/digit-recognizer/sample_submission.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 train = pd.read_csv(".. /input/digit-recognizer/train.csv" )<feature_engineering>
train_extended = pd.DataFrame(np.vstack(train_extended), columns = cols )
test['dataset'] = 'test'<feature_engineering>
train_extended = downcast(train_extended )
train['dataset'] = 'train'<concatenate>
train["revenue"] = train["item_price"] * train["item_cnt_day"]
dataset = pd.concat([train.drop('label', axis=1), test] ).reset_index()<concatenate>
train_monthly = train.groupby(['date_block_num', "shop_id", "item_id"], as_index = False)\ 
.agg({"item_cnt_day" : ["sum"]})
 train_monthly.columns = ['date_block_num',"shop_id", "item_id", "item_cnt_month"]
train_monthly = downcast(train_monthly )
mnist = pd.concat([mnist_train, mnist_test] ).reset_index(drop=True)
 labels = mnist['label'].values
mnist.drop('label', axis=1, inplace=True)
 mnist.columns = cols<sort_values>
train_monthly["item_cnt_month"] = train_monthly["item_cnt_month"].clip(0,20)
idx_mnist = mnist.sort_values(by=list(mnist.columns)).index
dataset_from = dataset.sort_values(by=list(mnist.columns)) ['dataset'].values
original_idx = dataset.sort_values(by=list(mnist.columns)) ['index'].values<feature_engineering>
train_new = train_extended.merge(train_monthly, on=["date_block_num", "shop_id", "item_id"], how="left" ).fillna(0 )
for i in range(len(idx_mnist)) :
if dataset_from[i] == 'test':
sample_submission.loc[original_idx[i], 'Label'] = labels[idx_mnist[i]]<save_to_csv>
train_new = downcast(train_new)
 train_new["item_cnt_month"] = train_new["item_cnt_month"].astype(np.int8 )
sample_submission.to_csv('submission.csv', index=False )<set_options>
test = test.drop("ID", axis = 1)
 test.insert(0, "date_block_num", 34 )
%matplotlib inline
np.random.seed(2)
 sns.set(style='white', context='notebook', palette='deep' )<load_from_csv>
test = downcast(test )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<feature_engineering>
train_new.fillna(0, inplace=True )
X_train = X_train / 255.0
test = test / 255.0<categorify>
train_new = downcast(train_new )
Y_train = to_categorical(Y_train, num_classes = 10 )<define_variables>
items_and_item_categories = items.drop(columns=["item_name"])\ 
.merge(item_categories.drop(columns=["item_category_name", "big_category_name", "sub_category_name"]))
items_and_item_categories.info()
random_seed = 2<split>
train_new = train_new.merge(items_and_item_categories, on = ["item_id"])
 test = test.merge(items_and_item_categories, on = ["item_id"] )
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=random_seed )<choose_model_class>
train_new = train_new.merge(shops.drop(columns=["shop_name", "city_name", "shop_category_name"]), on = ["shop_id"])
 test = test.merge(shops.drop(columns=["shop_name", "city_name", "shop_category_name"]), on = ["shop_id"] )
nets = 5
model = [0] *nets
for j in range(nets):
model[j] = Sequential()
model[j].add(Conv2D(32, kernel_size = 3, padding='same', activation='relu', input_shape =(28, 28, 1)))
 model[j].add(BatchNormalization())
 model[j].add(Conv2D(32, kernel_size = 3, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(MaxPooling2D(pool_size=(2, 2)))
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(64, kernel_size = 3, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64, kernel_size = 5, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(MaxPooling2D(pool_size=(2, 2)))
 model[j].add(Dropout(0.25))
model[j].add(Conv2D(128, kernel_size = 4, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(MaxPooling2D(pool_size=(2, 2)))
 model[j].add(Flatten())
 model[j].add(Dense(128, activation='relu'))
model[j].add(Dropout(0.7))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )<train_model>
train_new.fillna(0, inplace=True)
 test.fillna(0, inplace=True )
<init_hyperparams>
lags = [1,2,3,6,12]
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.3,
shear_range = 0.2,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<train_model>
train_new = lag_feature(train_new, lags, "item_cnt_month")
 train_new.fillna(0, inplace=True)
 train_new = downcast(train_new )
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 history = [0] * nets
epochs = 25
for j in range(nets):
X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.1)
 history[j] = model[j].fit_generator(datagen.flow(X_train2,Y_train2, batch_size=32),
epochs = epochs, steps_per_epoch = X_train2.shape[0]//32, verbose = 2,
validation_data =(X_val2,Y_val2), callbacks=[annealer])
 print("CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 j+1,epochs,max(history[j].history['accuracy']),max(history[j].history['val_accuracy'])) )<save_to_csv>
group = train_new.groupby("date_block_num" ).agg({"item_cnt_month": ["mean"]} ).reset_index()
group.columns = ["date_block_num", "date_block_item_cnt_mean"]
results = np.zeros(( test.shape[0],10))
for j in range(nets):
results = results + model[j].predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("MNIST-CNN-ENSEMBLE.csv",index=False )<load_from_csv>
train_new = train_new.merge(group, on=["date_block_num"])
 train_new.fillna(0, inplace=True )
df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 df2 = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 print(df.shape)
 print(df2.shape )<categorify>
train_new = lag_feature(train_new, [1], "date_block_item_cnt_mean")
 train_new.fillna(0, inplace=True )
x = np.array(df.drop('label',axis=1)) /255
y = np.array(df.label)
 test = np.array(df2)/255
enc = OneHotEncoder(sparse=False)
 y= y.reshape(( -1,1))
y = enc.fit_transform(y)
 print(x.shape)
 print(y.shape)
 print(test.shape )<split>
train_new.drop("date_block_item_cnt_mean", axis=1, inplace=True )
x_train,x_test,y_train,y_test = tts(x_2d,y,test_size = 0.15, random_state=42)
 print(x_train.shape)
 print(y_train.shape)
 print(x_test.shape)
 print(y_test.shape )<import_modules>
group = train_new.groupby(["date_block_num","shop_id"] ).agg({"item_cnt_month": ["mean"]} ).reset_index()
group.columns = ["date_block_num","shop_id", "shop_item_cnt_mean"]
train_new = train_new.merge(group, on=["date_block_num", "shop_id"])
 train_new = lag_feature(train_new, lags, "shop_item_cnt_mean")
 train_new.fillna(0, inplace=True )
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from keras import applications
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
from keras.callbacks import ReduceLROnPlateau<choose_model_class>
train_new.drop("shop_item_cnt_mean", axis=1, inplace=True )
model = Sequential()
model.add(Conv2D(32, kernel_size = 3, activation='relu', input_shape =(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Conv2D(128, kernel_size = 4, activation='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.summary() 
<init_hyperparams>
group = train_new.groupby(["date_block_num","item_id"] ).agg({"item_cnt_month": ["mean"]} ).reset_index()
group.columns = ["date_block_num","item_id", "item_item_cnt_mean"]
train_new = train_new.merge(group, on=["date_block_num", "item_id"])
 train_new = lag_feature(train_new, [1], "item_item_cnt_mean")
 train_new.drop("item_item_cnt_mean", axis=1, inplace=True)
 train_new.fillna(0, inplace=True )
callbacks = [
keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=1e-5,
patience=15,
verbose=1)
 ]<train_model>
group = train_new.groupby(["date_block_num","item_category_id"] ).agg({"item_cnt_month": ["mean"]} ).reset_index()
group.columns = ["date_block_num","item_category_id", "item_category_item_cnt_mean"]
train_new = train_new.merge(group, on=["date_block_num", "item_category_id"])
 train_new = lag_feature(train_new, [1], "item_category_item_cnt_mean")
 train_new.drop("item_category_item_cnt_mean", axis=1, inplace=True)
 train_new.fillna(0, inplace=True )
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 history = model.fit(x_2d,y,batch_size=64,epochs=600,validation_data=(x_test,y_test)) 
<compute_test_metric>
group = train_new.groupby(["date_block_num","city_id"] ).agg({"item_cnt_month": ["mean"]} ).reset_index()
group.columns = ["date_block_num","city_id", "city_item_cnt_mean"]
train_new = train_new.merge(group, on=["date_block_num", "city_id"])
 train_new = lag_feature(train_new, [1], "city_item_cnt_mean")
 train_new.drop("city_item_cnt_mean", axis=1, inplace=True)
 train_new.fillna(0, inplace=True )
scores = model.evaluate(x_test, y_test, verbose = 10)
 print(scores )<predict_on_test>
group = train_new.groupby(["date_block_num","big_category_id"] ).agg({"item_cnt_month": ["mean"]} ).reset_index()
group.columns = ["date_block_num","big_category_id", "big_category_item_cnt_mean"]
train_new = train_new.merge(group, on=["date_block_num", "big_category_id"])
 train_new = lag_feature(train_new, [1], "big_category_item_cnt_mean")
 train_new.drop("big_category_item_cnt_mean", axis=1, inplace=True)
 train_new.fillna(0, inplace=True )
predictions=model.predict(test_2d)
 pre=predictions.argmax(axis=-1 )<save_to_csv>
group = train_new.groupby(["date_block_num","sub_category_id"] ).agg({"item_cnt_month": ["mean"]} ).reset_index()
group.columns = ["date_block_num","sub_category_id", "sub_category_item_cnt_mean"]
train_new = train_new.merge(group, on=["date_block_num", "sub_category_id"])
 train_new = lag_feature(train_new, [1], "sub_category_item_cnt_mean")
 train_new.drop("sub_category_item_cnt_mean", axis=1, inplace=True)
 train_new.fillna(0, inplace=True )
submission = pd.Series(pre,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),submission],axis = 1)
 submission.to_csv("final_submission_lenet5.csv",index=False)
 submission.head()<import_modules>
group = train_new.groupby(["date_block_num","item_category_id", "item_id"] ).agg({"item_cnt_month": ["mean"]} ).reset_index()
group.columns = ["date_block_num","item_category_id", "item_id", "item_category_item_item_cnt_mean"]
train_new = train_new.merge(group, on=["date_block_num","item_category_id", "item_id"])
 train_new = lag_feature(train_new, lags, "item_category_item_item_cnt_mean")
 train_new.drop("item_category_item_item_cnt_mean", axis=1, inplace=True)
 train_new.fillna(0, inplace=True )
from sklearn.ensemble import RandomForestRegressor<import_modules>
group = train_new.groupby(["date_block_num", "shop_id", "item_category_id"] ).agg({"item_cnt_month": ["mean"]} ).reset_index()
group.columns = ["date_block_num", "shop_id", "item_category_id", "shop_item_category_item_cnt_mean"]
train_new = train_new.merge(group, on=["date_block_num", "shop_id", "item_category_id"])
 train_new = lag_feature(train_new, lags, "shop_item_category_item_cnt_mean")
 train_new.drop("shop_item_category_item_cnt_mean", axis=1, inplace=True)
 train_new.fillna(0, inplace=True )
from sklearn.ensemble import RandomForestRegressor<load_from_csv>
group = train_new.groupby(["date_block_num", "shop_id", "big_category_id"] ).agg({"item_cnt_month": ["mean"]} ).reset_index()
group.columns = ["date_block_num", "shop_id", "big_category_id", "shop_big_category_item_cnt_mean"]
train_new = train_new.merge(group, on=["date_block_num", "shop_id", "big_category_id"])
 train_new = lag_feature(train_new, [1], "shop_big_category_item_cnt_mean")
 train_new.drop("shop_big_category_item_cnt_mean", axis=1, inplace=True)
 train_new.fillna(0, inplace=True )
train = pd.read_csv("/kaggle/input/walmart-recruiting-store-sales-forecasting/train.csv.zip")
 test = pd.read_csv("/kaggle/input/walmart-recruiting-store-sales-forecasting/test.csv.zip")
 stores = pd.read_csv("/kaggle/input/walmart-recruiting-store-sales-forecasting/stores.csv" )<data_type_conversions>
group = train_new.groupby(["date_block_num", "shop_id", "sub_category_id"] ).agg({"item_cnt_month": ["mean"]} ).reset_index()
group.columns = ["date_block_num", "shop_id", "sub_category_id", "shop_sub_category_item_cnt_mean"]
train_new = train_new.merge(group, on=["date_block_num", "shop_id", "sub_category_id"])
 train_new = lag_feature(train_new, [1], "shop_sub_category_item_cnt_mean")
 train_new.drop("shop_sub_category_item_cnt_mean", axis=1, inplace=True)
 train_new.fillna(0, inplace=True )
train=train.merge(stores,how='left',on='Store')
 train.replace({'A': 1, 'B': 2,'C':3},inplace=True)
 train['Date']=pd.to_datetime(train['Date'])
 train['dia']=pd.to_datetime(train['Date'] ).dt.day
train['mes']=pd.to_datetime(train['Date'] ).dt.month
train['ano']=pd.to_datetime(train['Date'] ).dt.year
train.head()<feature_engineering>
train_new["month"] = train_new["date_block_num"] % 12
test=test.merge(stores,how='left',on='Store')
 test.replace({'A': 1, 'B': 2,'C':3},inplace=True)
 test['Date']=pd.to_datetime(test['Date'])
 test['dia']=pd.to_datetime(test['Date'] ).dt.day
test['mes']=pd.to_datetime(test['Date'] ).dt.month
test['ano']=pd.to_datetime(test['Date'] ).dt.year
test.head()<predict_on_test>
holidays = [13, 8, 11, 8, 10, 11, 8, 9, 9, 8, 10, 9,
12, 8, 10, 4, 11, 10, 8, 10, 8, 8, 12, 8,
13, 9, 9, 8, 11, 9, 8, 10, 8, 9, 11]
holidays_dict = dict(zip(list(range(35)) , holidays))
print(holidays_dict)
 train_new["holidays_cnt"] = train_new["date_block_num"].map(holidays_dict )
X = train.loc[:, train.columns != 'Weekly_Sales'].drop('Date',axis=1)
 y = train.loc[:, train.columns == 'Weekly_Sales']
rf = RandomForestRegressor()
rf.fit(X, y)
 test['Weekly_Sales'] = rf.predict(test.drop('Date',axis=1))<save_to_csv>
rates = [
0.033064, 0.033135, 0.032450, 0.031901, 0.031923, 0.030951, 0.030517, 0.030310, 0.030674, 0.031174, 0.030592, 0.030425,
0.029695, 0.028373, 0.027628, 0.028044, 0.028638, 0.029076, 0.028831, 0.027667, 0.026349, 0.024501, 0.021618, 0.017868,
0.015704, 0.015517, 0.016594, 0.018804, 0.019731, 0.018311, 0.017496, 0.015301, 0.014936, 0.015841, 0.015365
]
rates_dict = dict(zip(list(range(35)) , rates))
train_new["dollar_ruble_rate"] = train_new["date_block_num"].map(rates_dict )
test['ID']=test['Store'].astype(str)+'_'+test['Dept'].astype(str)+'_'+test['Date'].astype(str)
 test[['ID','Weekly_Sales']].to_csv('sub.csv',index=False )<load_from_csv>
days = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
days_dict = dict(zip(list(range(13)) , rates))
train_new["days_cnt"] = train_new["month"].map(days_dict )
train=pd.read_csv("/kaggle/input/walmart-recruiting-store-sales-forecasting/train.csv")
 train.head()<load_from_csv>
test_sort = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 test_sort.drop("ID", axis=1, inplace = True)
 train = downcast(train_new.query('date_block_num < 31 and date_block_num > 5'))
val = downcast(train_new.query('date_block_num > 30 and date_block_num < 34'))
test = downcast(test_sort.merge(train_new.query('date_block_num == 34'), on = ["shop_id", "item_id"], how="left"))
test=pd.read_csv("/kaggle/input/walmart-recruiting-store-sales-forecasting/test.csv")
 test.head()<prepare_x_and_y>
train = downcast(train)
 val = downcast(val)
 test = downcast(test )
y=train["Weekly_Sales"]<data_type_conversions>
X_train = train.drop("item_cnt_month", axis = 1 ).reset_index(drop=True)
 X_val = val.drop("item_cnt_month", axis = 1 ).reset_index(drop=True)
 y_train = train[["item_cnt_month"]].reset_index(drop=True)
 y_val = val[["item_cnt_month"]].reset_index(drop=True)
 X_test = test.drop("item_cnt_month", axis = 1 ).reset_index(drop=True )
train["Date"]=train["Date"].astype("datetime64")
 test["Date"]=test["Date"].astype("datetime64")
 train.dtypes<feature_engineering>
X_train = downcast(X_train)
 X_val = downcast(X_val)
 y_train = downcast(y_train)
 y_val = downcast(y_val)
 X_test = downcast(X_test )
train["month"]=train["Date"].dt.month
test["month"]=test["Date"].dt.month
train["day"]=train["Date"].dt.day
test["day"]=test["Date"].dt.day
train["year"]=train["Date"].dt.year
test["year"]=test["Date"].dt.year<load_from_csv>
del train_new
del train
del test
del val
del items
del shops
del item_categories
store=pd.read_csv("/kaggle/input/walmart-recruiting-store-sales-forecasting/stores.csv")
 store.head()<merge>
gc.collect()
train=pd.merge(train,store)
 test=pd.merge(test,store )<categorify>
xgb_features = ['date_block_num',
'shop_id',
'item_id',
'item_category_id',
'big_category_id',
'sub_category_id',
'city_id',
'shop_category_id',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'date_block_item_cnt_mean_lag_1',
'shop_item_cnt_mean_lag_1',
'shop_item_cnt_mean_lag_2',
'shop_item_cnt_mean_lag_3',
'item_item_cnt_mean_lag_1',
'item_category_item_cnt_mean_lag_1',
'big_category_item_cnt_mean_lag_1',
'sub_category_item_cnt_mean_lag_1',
'shop_category_item_cnt_mean_lag_1',
'item_category_item_item_cnt_mean_lag_1',
'item_category_item_item_cnt_mean_lag_2',
'shop_item_category_item_cnt_mean_lag_1',
'shop_item_category_item_cnt_mean_lag_2',
'shop_big_category_item_cnt_mean_lag_1',
'shop_sub_category_item_cnt_mean_lag_1',
'month',
'holidays_cnt',
'dollar_ruble_rate',
'days_cnt']
train["Type"]=train["Type"].replace({"A":1,"B":2,"C":3})
 test["Type"]=test["Type"].replace({"A":1,"B":2,"C":3} )<drop_column>
X_train_xgb = X_train.loc[:, xgb_features]
X_val_xgb = X_val.loc[:, xgb_features]
X_test_xgb = X_test.loc[:, xgb_features]
train=train.drop(["Weekly_Sales","Date"],axis=1)
 train.head()<drop_column>
X_train_xgb.fillna(0, inplace=True)
 X_val_xgb.fillna(0, inplace=True)
 X_test_xgb.fillna(0, inplace=True )
tesst=test.drop(["Date"],axis=1)
 test.head()<train_model>
xgb_model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=400,
colsample_bytree=0.6,
subsample=0.6,
eta=0.2,
seed=0,
learning_rate = 0.1,
n_jobs=-1)
 xgb_model.fit(
 X_train_xgb,
y_train,
eval_metric="rmse",
eval_set=[(X_train_xgb, y_train),(X_val_xgb, y_val)],
verbose=10,
early_stopping_rounds = 10 )
rf=RandomForestRegressor(n_estimators=100, n_jobs=-1)
 rf.fit(train,y )<predict_on_test>
xgb_train_pred = xgb_model.predict(X_train_xgb)
 xgb_val_pred = xgb_model.predict(X_val_xgb)
 xgb_test_pred = xgb_model.predict(X_test_xgb )
value=rf.predict(test )<load_from_csv>
rf_features = ['date_block_num',
'shop_id',
'item_id',
'item_category_id',
'big_category_id',
'sub_category_id',
'city_id',
'shop_category_id',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'date_block_item_cnt_mean_lag_1',
'shop_item_cnt_mean_lag_1',
'shop_item_cnt_mean_lag_2',
'item_item_cnt_mean_lag_1',
'item_category_item_cnt_mean_lag_1',
'big_category_item_cnt_mean_lag_1',
'sub_category_item_cnt_mean_lag_1',
'shop_category_item_cnt_mean_lag_1',
'item_category_item_item_cnt_mean_lag_1',
'shop_item_category_item_cnt_mean_lag_1',
'shop_item_category_item_cnt_mean_lag_2',
'shop_big_category_item_cnt_mean_lag_1',
'shop_sub_category_item_cnt_mean_lag_1',
'month',
'holidays_cnt',
'dollar_ruble_rate',
'days_cnt']
sample=pd.read_csv("/kaggle/input/walmart-recruiting-store-sales-forecasting/sampleSubmission.csv")
 sample.head()<feature_engineering>
X_train_rf = X_train.loc[:, rf_features]
X_val_rf = X_val.loc[:, rf_features]
X_test_rf = X_test.loc[:, rf_features]
sample["Weekly_Sales"]=value<save_to_csv>
X_train_rf.fillna(0, inplace=True)
 X_val_rf.fillna(0, inplace=True)
 X_test_rf.fillna(0, inplace=True )
sample.to_csv("20191014.csv",index=False )<set_options>
rf_model = RandomForestRegressor(n_estimators=50, max_depth=7, random_state=0, n_jobs=-1)
 rf_model.fit(X_train_rf, y_train )
tf.random.set_seed(42)
<load_from_csv>
rf_train_pred = rf_model.predict(X_train_rf)
 rf_val_pred = rf_model.predict(X_val_rf)
 rf_test_pred = rf_model.predict(X_test_rf.fillna(0))
train = dtable.fread('/kaggle/input/jane-street-market-prediction/train.csv' ).to_pandas()<prepare_x_and_y>
first_level = pd.DataFrame(xgb_val_pred, columns=["xgb"])
 first_level["rf"] = rf_val_pred
first_level.info()
first_level_test = pd.DataFrame(xgb_test_pred, columns=["xgb"])
 first_level_test["rf"] = rf_test_pred
first_level_test.info()
features = [c for c in train.columns if 'feature' in c]
f_mean = train.loc[:, features[1:]].mean()
train = train.query('weight > 0' ).reset_index(drop = True)
 train = train.query('date > 85' ).reset_index(drop = True)
 train.loc[:, features[1:]] = train.loc[:, features[1:]].fillna(f_mean)
 train.loc[:, 'action'] =(train.loc[:, 'resp'] > 0 ).astype('int')
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
f_mean = f_mean.values
np.save('f_mean.npy', f_mean)
 X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T<import_modules>
meta_model = LinearRegression(n_jobs=-1)
 meta_model.fit(first_level, y_val )
class GroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_size=None 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_size = max_train_size
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = n_groups // n_folds
group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
for train_group_idx in unique_groups[:group_test_start]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
if self.max_train_size and self.max_train_size < train_end:
train_array = train_array[train_end -
self.max_train_size:train_end]
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 yield [int(i)for i in train_array], [int(i)for i in test_array] 
<init_hyperparams>
test_prediction = meta_model.predict(first_level_test )
class LearningRateLogger(tf.keras.callbacks.Callback):
def __init__(self):
super().__init__()
self._supports_tf_logs = True
def on_epoch_end(self, epoch, logs=None):
if logs is None or "learning_rate" in logs:
return
logs["learning_rate"] = self.model.optimizer.lr
def get_callbacks(name, fold):
return [
ReduceLROnPlateau(monitor = 'val_AUC', factor = 0.5, patience = 5, verbose = 0, min_delta = 1e-4, mode = 'max'),
ModelCheckpoint(f'JSModel_{fold}.hdf5' , monitor = 'val_AUC', verbose = 0, save_best_only = True,
save_weights_only = True, mode = 'max'),
EarlyStopping(monitor = 'val_AUC', min_delta = 1e-4, patience = 15, mode = 'max',
baseline = None, restore_best_weights = True, verbose = 0),
LearningRateLogger() ,
]<compute_test_metric>
submission = pd.DataFrame(test_prediction, columns=["item_cnt_month"] ).clip(0, 20 ).reset_index()
submission.columns = ["ID", "item_cnt_month"]
print(submission )
def utility_score_bincount(date, weight, resp, action):
count_i = len(np.unique(date))
Pi = np.bincount(date, weight * resp * action)
 t = np.sum(Pi)/ np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / count_i)
 u = np.clip(t, 0, 6)* np.sum(Pi)
 return u<define_variables>
submission.to_csv("submission.csv", index=False )
<feature_engineering><EOS> 
submission.to_csv("submission.csv", index=False )