Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<choose_model_class>
BASE_DIR= ".. /input/"
model=make_model()
model.compile(optimizer=optimizers.Adam(lr=0.0001), loss='categorical_crossentropy',metrics=['accuracy'])
 model.summary()<train_model>
salesData= pd.read_csv(f"{BASE_DIR}sales_train.csv")
 items= pd.read_csv(f"{BASE_DIR}items.csv")
 testDF= pd.read_csv(f"{BASE_DIR}test.csv")
 itemCategories= pd.read_csv(f"{BASE_DIR}item_categories.csv")
 shops= pd.read_csv(f"{BASE_DIR}shops.csv")
 salesData.drop(['date'],axis=1,inplace=True )
model.fit(x=X_train, y=y_train, batch_size=32, epochs=15 )<predict_on_test>
uniqueTestItems = testDF['item_id'].nunique()
print("Number of unique test items = "+str(uniqueTestItems))
uniqueTrainItems = salesData['item_id'].nunique()
print("Number of unique train items = "+str(uniqueTrainItems))
pred = model.predict(X_test,verbose=1)
 predictions = pred.argmax(axis=-1 )<load_from_csv>
uniqueTestShops = testDF['shop_id'].nunique()
print("Number of unique test shops = "+str(uniqueTestShops))
uniqueTrainShops = salesData['shop_id'].nunique()
print("Number of unique train shops = "+str(uniqueTrainShops))
sub = pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')
 submission = sub['ImageId']<merge>
uniqueTrainItems = salesData['item_id'].unique()
itemsNotInTrainDF = testDF[ ~testDF['item_id'].isin(uniqueTrainItems)]
itemsInTrainDF = testDF[ testDF['item_id'].isin(uniqueTrainItems)]
itemsNotInTrainDF.shape[0]/itemsInTrainDF.shape[0]
pred = pd.DataFrame(data=predictions ,columns=["Label"])
 DT = pd.merge(submission , pred, on=None, left_index= True,
right_index=True)
 DT.head()<save_to_csv>
tmpCat= itemCategories.copy()
tmpCat['isHyphenInName']= tmpCat['item_category_name'].apply(lambda x: 1 if '-' in x else 0)
 tmpCat['isHyphenInName'].sum() /tmpCat.shape[0]
DT.to_csv('submission.csv',index = False )<set_options>
tmpShops= shops.copy()
tmpShops['firstWordInName']= tmpShops['shop_name'].apply(lambda x: x.split(' ')[0])
 tmpShops['firstWordInName'].nunique() /tmpShops.shape[0]
warnings.simplefilter("ignore" )<categorify>
salesData= filteredTrainDataWithItemPair
salesData=salesData[(salesData['item_price']<50000)&(salesData['item_cnt_day']<500)]
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<choose_model_class>
targetDF= salesData.groupby(['item_id','shop_id','date_block_num'],as_index=False ).agg({'item_cnt_day':['sum'],'item_price':'mean'})
 targetDF.columns= ['item_id','shop_id','date_block_num','target','avg_price']
def build_model(transformer, loss='binary_crossentropy', max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
x = tf.keras.layers.Dropout(0.35 )(cls_token)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=3e-5), loss=loss, metrics=[tf.keras.metrics.AUC() ])
 return model<string_transform>
testDF['date_block_num'] = 34
testDF['date_block_num'] = testDF['date_block_num'].astype(np.int8)
 testDF['shop_id'] = testDF['shop_id'].astype(np.int8)
 testDF['item_id'] = testDF['item_id'].astype(np.int16)
 cartesianDF = pd.concat([cartesianDF, testDF], ignore_index=True, sort=False, keys=['shop_id','item_id','date_block_num'] )
nltk.download('punkt')
 stemmer = nltk.stem.porter.PorterStemmer()
remove_punctuation_map = dict(( ord(char), None)for char in string.punctuation)
 def stem_tokens(tokens):
return [stemmer.stem(item)for item in tokens]

 def normalize(text):
return stem_tokens(nltk.word_tokenize(text.lower().translate(remove_punctuation_map)))
 vectorizer = TfidfVectorizer(tokenizer=normalize, stop_words='english')
 def cosine_sim(text1, text2):
tfidf = vectorizer.fit_transform([text1, text2])
 return(( tfidf * tfidf.T ).A)[0,1]<choose_model_class>
%%time
resDF= cartesianDF.merge(targetDF,how='left',on=['shop_id','item_id','date_block_num'] )
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
<load_pretrained>
itemPrice= salesData.groupby(['item_id'],as_index=False ).agg({'item_price':'mean'})
 itemPrice.columns=['item_id','avg_price_item']
resDF['avg_price'][ resDF['avg_price'].isna() ]= resDF[ resDF['avg_price'].isna() ].merge(itemPrice,how='left',on=['item_id'])['avg_price_item'].values
tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-cased')
 save_path = '/kaggle/working/distilbert_base_uncased/'
if not os.path.exists(save_path):
os.makedirs(save_path)
 tokenizer.save_pretrained(save_path)
 fast_tokenizer = BertWordPieceTokenizer('distilbert_base_uncased/vocab.txt', lowercase=False)
 fast_tokenizer<load_from_csv>
shops['shop_label']= shops['shop_name'].apply(lambda x: x.split(' ')[0])
 le = preprocessing.LabelEncoder()
shops['shop_label']= le.fit_transform(shops['shop_label'] )
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 valid = pd.read_csv('/kaggle/input/val-en-df/validation_en.csv')
 test1 = pd.read_csv('/kaggle/input/test-en-df/test_en.csv')
 test2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<prepare_x_and_y>
itemCategories['cat_label']= itemCategories['item_category_name'].apply(lambda x: x.split('-')[0].strip())
 itemCategories['cat_subLabel']= itemCategories['item_category_name'].apply(lambda x: x.split('-')[1].strip() if '-' in x else x.split('-')[0].strip())
 le = preprocessing.LabelEncoder()
itemCategories['cat_label']= le.fit_transform(itemCategories['cat_label'])
 itemCategories['cat_subLabel']= le.fit_transform(itemCategories['cat_subLabel'] )
x_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=512)
 x_valid = fast_encode(valid.comment_text_en.astype(str), fast_tokenizer, maxlen=512)
 x_test1 = fast_encode(test1.content_en.astype(str), fast_tokenizer, maxlen=512)
 x_test2 = fast_encode(test2.translated.astype(str), fast_tokenizer, maxlen=512)
 y_train = train1.toxic.values
y_valid = valid.toxic.values<categorify>
resDF= resDF.merge(items[['item_id','item_category_id']],on=['item_id'],how='left')
 resDF= resDF.merge(itemCategories[['cat_label','cat_subLabel','item_category_id']],how='left',on=['item_category_id'])
 resDF= resDF.merge(shops[['shop_label','shop_id']],how='left',on=['shop_id'] )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(64)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(64)
.cache() 
.prefetch(AUTO)
)
 test_dataset = [(
 tf.data.Dataset 
.from_tensor_slices(x_test1)
.batch(64)
), 
(
 tf.data.Dataset 
.from_tensor_slices(x_test2)
.batch(64)
)]<compute_train_metric>
meanEncodingCols =[ 
(['shop_id'],[1,2,3,6,12]), 
(['item_id'],[1,2,3,6,12]), 
(['item_category_id'],[1,2,3,6,12]), 
(['item_id','shop_id'],[1,2,3,6,12]), 
(['item_category_id','shop_id'],[1,2,3,6,12]), 
(['cat_label'],[1]), 
(['cat_label','item_id'],[1,2,3]), 
(['cat_subLabel'],[1]), 
(['shop_label'],[1])
 ]
def focal_loss(gamma=2., alpha=.2):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) - K.mean(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return focal_loss_fixed<load_pretrained>
def generateGroupByFeatures(ipDF,groupCols,aggDict):
groupByAlias = {'item_id':'item','shop_id':'shop','item_category_id':'item_cat','date_block_num':'date',
'month':'month','target':'target','cat_label':'cat_label','cat_subLabel':'cat_subLabel',
'shop_label':'shop_label'}
aggColAlias= {'item_cnt_day':'cnt','item_price':'price','target':'target','avg_price':'price'}
grpByName= '_'.join([groupByAlias[x] for x in groupCols])+'_'
colNameList=[]
for aggCol,aggFnArr in aggDict.items() :
for f in aggFnArr:
fName = f.__name__ if hasattr(f, '__call__')else f
colNameList.append(grpByName+aggColAlias[aggCol]+"_"+fName)
 returnDF= ipDF.groupby(groupCols,as_index=False ).agg(aggDict)
 returnDF.columns= groupCols+colNameList
return returnDF
%%time
with strategy.scope() :
transformer_layer = transformers.TFBertModel.from_pretrained('bert-base-cased')
 model = build_model(transformer_layer, loss=focal_loss(gamma=1.5), max_len=512)
 model.summary()<choose_model_class>
for c in resDF.columns:
if 'int' in str(resDF[c].dtype):
resDF[c]= resDF[c].astype('Int32')
 if resDF[c].max() <32000:
resDF[c]= resDF[c].astype('Int16')
 if resDF[c].max() <120:
resDF[c]= resDF[c].astype('Int8')
 if 'float' in str(resDF[c].dtype):
resDF[c]= resDF[c].astype('Float32')
 if resDF[c].max() <65000:
resDF[c]= resDF[c].astype('Float16' )
class RocAucCallback(Callback):
def __init__(self, test_data, score_thr):
self.test_data = test_data
self.score_thr = score_thr
self.test_pred = []
def on_epoch_end(self, epoch, logs=None):
if logs['val_auc'] > self.score_thr:
print('
Run TTA...')
 for td in self.test_data:
self.test_pred.append(self.model.predict(td))<choose_model_class>
grpByItem= generateGroupByFeatures(salesData,['item_id'],{'item_price':['min','max',np.var]})
 grpByItem['historic_price_shift']=(grpByItem['item_price_max']-grpByItem['item_price_min'])/grpByItem['item_price_max']
grpByItem.drop(['item_price_max','item_price_min'],axis=1,inplace=True)
 resDF= resDF.merge(grpByItem,how='left',on=['item_id'] )
def build_lrfn(lr_start=0.000001, lr_max=0.000004,
lr_min=0.0000001, lr_rampup_epochs=7,
lr_sustain_epochs=0, lr_exp_decay=.87):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn<train_on_grid>
%%time
for m in meanEncodingCols:
grpCols= m[0]
lagArr= m[1]
tmpDF= resDF.copy()
grpCols= grpCols+['date_block_num']
groupDF= generateGroupByFeatures(tmpDF,grpCols,{'target':['mean']})
 tmpDF= tmpDF.merge(groupDF,how='left',on= grpCols)
 meanEncodedColName= [c for c in groupDF.columns if 'target_mean' in c]
assert len(meanEncodedColName)==1
meanEncodedColName= meanEncodedColName[0]
for lag in lagArr:
shiftedDF= tmpDF[['item_id','shop_id','date_block_num',meanEncodedColName]].copy()
shiftedDF['date_block_num']= shiftedDF['date_block_num']+lag
lagColName= meanEncodedColName+"_lag_"+str(lag)
 shiftedDF.rename({meanEncodedColName:lagColName},axis=1,inplace=True)
 resDF= resDF.merge(shiftedDF,how='left',on=['item_id','shop_id','date_block_num'])
 del shiftedDF
del tmpDF
roc_auc = RocAucCallback(test_dataset, 0.935)
 lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 train_history = model.fit(
 train_dataset,
steps_per_epoch=150,
validation_data=valid_dataset,
callbacks=[lr_schedule, roc_auc],
epochs=35
)<import_modules>
%%time
for lag in [1,2,3]:
tmpDF= resDF[['item_id','date_block_num','avg_price']].copy()
tmpDF['date_block_num']+= lag
tmpGrpByItem= tmpDF.groupby(['item_id','date_block_num'],as_index=False ).agg({'avg_price':['mean']})
 tmpGrpByItem.columns= ['item_id','date_block_num','item_price_lag'+str(lag)]
resDF= resDF.merge(tmpGrpByItem,how='left',on=['item_id','date_block_num'])
 del tmpGrpByItem
del tmpDF
from sklearn.model_selection import train_test_split<import_modules>
%%time
for lag in [1,2,3]:
tmpDF= resDF[['item_category_id','date_block_num','avg_price']].copy()
tmpDF['date_block_num']+= lag
tmpGrpByItem= tmpDF.groupby(['item_category_id','date_block_num'],as_index=False ).agg({'avg_price':['mean']})
 tmpGrpByItem.columns= ['item_category_id','date_block_num','item_cat_price_lag'+str(lag)]
resDF= resDF.merge(tmpGrpByItem,how='left',on=['item_category_id','date_block_num'])
 del tmpGrpByItem
del tmpDF
from sklearn.model_selection import train_test_split<split>
for col in resDF.columns:
if 'price' not in col and(resDF[col].isnull().any()):
resDF[col].fillna(0, inplace=True )
x_train, x_valid, y_train, y_valid = train_test_split(x_valid, y_valid, test_size=0.15, shuffle=True, random_state=123, stratify=y_valid)
 train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(64)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(64)
.cache() 
.prefetch(AUTO)
 )<train_model>
for col in resDF.columns:
if 'price' in col:
resDF[col]= resDF[col].fillna(resDF[col].median() )
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 train_history = model.fit(
 train_dataset,
steps_per_epoch=75,
validation_data=valid_dataset,
callbacks=[lr_schedule, roc_auc],
epochs=25
)<save_to_csv>
gc.collect()
resDF.to_pickle('data.pkl' )
sub['toxic'] = np.mean(roc_auc.test_pred, axis=0)
 sub.to_csv('submission.csv', index=False )<import_modules>
resDF['month']= resDF['date_block_num']%12
resDF=resDF[ resDF['date_block_num']>12 ]
PATH = '.. /input/pku-autonomous-driving/'
os.listdir(PATH )<define_variables>
for c in ['item_price_lag1', 'item_price_lag2','item_price_lag3']:
resDF[c+'_diff']= resDF[c]/resDF['avg_price']
resDF.drop([c],axis=1,inplace=True )
SWITCH_LOSS_EPOCH = 5
print(torch.__version__ )<import_modules>
%%time
X= resDF[resDF['date_block_num']<33]
Y= X['target']
X.drop(['target'],axis=1,inplace=True)
 X_val= resDF[(resDF['date_block_num']==33)&(resDF['target']==0)]
Y_val= X_val['target']
X_val.drop(['target'],axis=1,inplace=True)
 X_val_1= resDF[(resDF['date_block_num']==33)&(resDF['target']!=0)]
Y_val_1= X_val_1['target']
X_val_1.drop(['target'],axis=1,inplace=True)
 model = xgb.XGBRegressor(max_depth=5 ,
n_estimators=100,
min_child_weight=20,
colsample_bytree=0.6,
seed=42,
n_jobs=-1)
 model.fit(X,Y,eval_metric="rmse",
eval_set=[(X,Y),(X_val, Y_val),(X_val_1, Y_val_1)],
verbose=20,
early_stopping_rounds=10)
 pickle.dump(model, open("xgb.pickle.dat", "wb"))
from keras.applications.inception_resnet_v2 import InceptionResNetV2<load_from_csv>
%%time
X= resDF[resDF['date_block_num']<33]
Y= X['target']
X.drop(['target'],axis=1,inplace=True)
 X_val= resDF[(resDF['date_block_num']==33)]
Y_val= X_val['target']
X_val.drop(['target'],axis=1,inplace=True)
 lr = LinearRegression(n_jobs=-1)
 lr.fit(X.values, Y.values)
 pickle.dump(lr, open("lr.pickle.dat", "wb"))
train = pd.read_csv(PATH + 'train.csv')
 test = pd.read_csv(PATH + 'sample_submission.csv')
 bad_list = ['ID_1a5a10365',
'ID_1db0533c7',
'ID_53c3fe91a',
'ID_408f58e9f',
'ID_4445ae041',
'ID_bb1d991f6',
'ID_c44983aeb',
'ID_f30ebe4d4']
train = train.loc[~train['ImageId'].isin(bad_list)]
camera_matrix = np.array([[2304.5479, 0, 1686.2379],
[0, 2305.8757, 1354.9849],
[0, 0, 1]], dtype=np.float32)
 camera_matrix_inv = np.linalg.inv(camera_matrix)
 train.head()<categorify>
X_train= resDF[resDF['date_block_num']<33].drop(['target'],axis=1)
 X_test= resDF[resDF['date_block_num']==34].drop(['target'],axis=1)
 X_val= resDF[resDF['date_block_num']==33].drop(['target'],axis=1)
 Y_train= resDF[resDF['date_block_num']<33]['target']
Y_val= resDF[resDF['date_block_num']==33]['target']
xgbModel= model;lrModel= lr;
def str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):

 coords = []
for l in np.array(s.split() ).reshape([-1, 7]):
coords.append(dict(zip(names, l.astype('float'))))
if 'id' in coords[-1]:
coords[-1]['id'] = int(coords[-1]['id'])
 return coords<train_model>
meta_xg_train= xgbModel.predict(X_train)
 meta_lr_train= lrModel.predict(X_train)
 meta_xg_val= xgbModel.predict(X_val)
 meta_lr_val= lrModel.predict(X_val)
 meta_xg_test= xgbModel.predict(X_test)
 meta_lr_test= lrModel.predict(X_test)
 metaTrain= np.c_[meta_xg_train,meta_lr_train]
print(metaTrain.shape,Y_train.shape)
 metaVal= np.c_[meta_xg_val,meta_lr_val]
print(metaVal.shape,Y_val.shape)
 metaTest= np.c_[meta_xg_test,meta_lr_test]
print(metaTest.shape )
inp = train['PredictionString'][0]
print('Example input:
', inp)
 print()
print('Output:
', str2coords(inp))<categorify>
lr= LinearRegression(n_jobs=-1)
 lr.fit(metaTrain,Y_train.values)
 metaVal= np.c_[meta_xg_val,meta_lr_val]
Y_train_pred= lr.predict(metaTrain)
 Y_val_pred= lr.predict(metaVal)
 print(np.sqrt(mean_squared_error(Y_val_pred,Y_val.values)))
 print(np.sqrt(mean_squared_error(Y_train_pred,Y_train.values)) )
def get_img_coords(s):

 coords = str2coords(s)
 xs = [c['x'] for c in coords]
ys = [c['y'] for c in coords]
zs = [c['z'] for c in coords]
P = np.array(list(zip(xs, ys, zs)) ).T
img_p = np.dot(camera_matrix, P ).T
img_p[:, 0] /= img_p[:, 2]
img_p[:, 1] /= img_p[:, 2]
img_xs = img_p[:, 0]
img_ys = img_p[:, 1]
img_zs = img_p[:, 2]
return img_xs, img_ys
plt.figure(figsize=(14,14))
plt.imshow(imread(PATH + 'train_images/' + train['ImageId'][2217] + '.jpg'))
plt.scatter(*get_img_coords(train['PredictionString'][2217]), color='red', s=100);<prepare_x_and_y>
drop_cols= [ 'target']
testDF = resDF[ resDF['date_block_num']==34]
testDF['target']= lr.predict(metaTest)
 submission = pd.DataFrame({
"ID": testDF['ID'],
"item_cnt_month": testDF['target'].clip(0,20)
 })
 submission['ID']= submission['ID'].astype(np.int32)
 submission.to_csv('submission.csv', index=False )