Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<drop_column>
sys.path.append('.. /input/iterativestratification')
 warnings.filterwarnings('ignore')
 os.listdir('.. /input/lish-moa')
 pd.set_option('max_columns', 2000 )
del train_match
del sale_record
del item
del shop
del ic
del train
del grouped
del grouped2
del grouped3
del grouped4
del grouped5
del grouped6
del grouped7
del grouped8
del grouped9
del grouped10
del grouped11
gc.collect()<load_pretrained>
n_comp_GENES = 463
n_comp_CELLS = 60
VarianceThreshold_for_FS = 0.9
Dropout_Model = 0.25
print('n_comp_GENES', n_comp_GENES, 'n_comp_CELLS', n_comp_CELLS, 'total', n_comp_GENES + n_comp_CELLS )
training = pd.read_pickle('.. /input/training-data-for-item-sale-predict/training.pkl' )<prepare_x_and_y>
commits_df = pd.DataFrame(columns = ['n_commit', 'n_comp_GENES', 'n_comp_CELLS', 'train_features','VarianceThreshold_for_FS', 'Dropout_Model', 'LB_score', 'CV_logloss'] )
train_x = training[training.date_block_num<33].drop(['item_cnt_month'],axis=1)
 train_y = training[training.date_block_num<33]['item_cnt_month']
valid_x = training[training.date_block_num==33].drop(['item_cnt_month'],axis=1)
 valid_y = training[training.date_block_num==33]['item_cnt_month']
test_x = training[training.date_block_num==34].drop(['item_cnt_month'],axis=1 )<choose_model_class>
n=0
commits_df.loc[n, 'n_commit'] = 0
commits_df.loc[n, 'n_comp_GENES'] = 600
commits_df.loc[n, 'n_comp_CELLS'] = 50
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.8
commits_df.loc[n, 'train_features'] = 1245
commits_df.loc[n, 'Dropout_Model'] = 0.2619422201258426
commits_df.loc[n, 'CV_logloss'] = 0.01458269555140327
commits_df.loc[n, 'LB_score'] = 0.01839
max_depth_range = {'max_depth':range(6,10,1)}
cv1 = GridSearchCV(estimator = XGBRegressor(n_estimators = 1000,
eta = 0.3,
subsample=0.8,
colsample_bytree=0.8,
seed = 0),
param_grid = max_depth_range,
scoring = 'neg_root_mean_squared_error')
 cv1.fit(train_x,train_y)
 cv1.cv_results_, cv1.best_score_, cv1.best_params_<choose_model_class>
n=1
commits_df.loc[n, 'n_commit'] = 4
commits_df.loc[n, 'n_comp_GENES'] = 610
commits_df.loc[n, 'n_comp_CELLS'] = 55
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.82
commits_df.loc[n, 'train_features'] = 1240
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014584545081734047
commits_df.loc[n, 'LB_score'] = 0.01839
xgb_model = XGBRegressor(n_estimators = 1500,
max_depth = 8,
eta = 0.1,
subsample=0.8,
colsample_bytree=0.8,
min_child_weight = 3,
seed = 42 )<train_on_grid>
n=2
commits_df.loc[n, 'n_commit'] = 5
commits_df.loc[n, 'n_comp_GENES'] = 670
commits_df.loc[n, 'n_comp_CELLS'] = 67
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.67
commits_df.loc[n, 'train_features'] = 1298
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014588561242139069
commits_df.loc[n, 'LB_score'] = 0.01840
xgb_param = xgb_model.get_xgb_params()
xg_train = xgb.DMatrix(train_x, train_y)
 cv_res = xgb.cv(xgb_param, xg_train, num_boost_round =1000, nfold =3,
metrics ='rmse', early_stopping_rounds =10, stratified =True,
seed=0)
 cv_res<train_model>
n=3
commits_df.loc[n, 'n_commit'] = 6
commits_df.loc[n, 'n_comp_GENES'] = 450
commits_df.loc[n, 'n_comp_CELLS'] = 45
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.67
commits_df.loc[n, 'train_features'] = 1297
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014586229676302227
commits_df.loc[n, 'LB_score'] = 0.01840
xgb_model.fit(train_x,train_y,
eval_metric = 'rmse',
eval_set = [(train_x, train_y),(valid_x, valid_y)],
early_stopping_rounds = 10 )<save_to_csv>
n=4
commits_df.loc[n, 'n_commit'] = 9
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1219
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014572358066092783
commits_df.loc[n, 'LB_score'] = 0.01839
result = xgb_model.predict(test_x ).clip(0,20)
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 submission = pd.DataFrame({'ID':test.index, 'item_cnt_month':result})
 submission.to_csv('xgb_submission.csv',index =False )<compute_test_metric>
n=5
commits_df.loc[n, 'n_commit'] = 10
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 80
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.92
commits_df.loc[n, 'train_features'] = 1214
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014571552074579226
commits_df.loc[n, 'LB_score'] = 0.01841
def rmse(y_pred, y_true):
return K.sqrt(K.mean(K.square(y_pred - y_true)))
 def MLP_model() :
model = Sequential()
model.add(Dense(256, activation ='relu',input_dim =30))
model.add(Dropout(0.2))
model.add(Dense(128, activation ='relu'))
model.add(Dropout(0.2))
model.add(Dense(1))
adam = Adam(lr =5e-5)
 model.compile(loss ='mse', optimizer =adam, metrics =[rmse])
 return model<train_model>
n=6
commits_df.loc[n, 'n_commit'] = 12
commits_df.loc[n, 'n_comp_GENES'] = 450
commits_df.loc[n, 'n_comp_CELLS'] = 65
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1219
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.01458043214513875
commits_df.loc[n, 'LB_score'] = 0.01840
base_model = MLP_model()
base_model.fit(train_x, train_y, batch_size =32, epochs =15, validation_data =(valid_x, valid_y))<save_model>
n=7
commits_df.loc[n, 'n_commit'] = 13
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1219
commits_df.loc[n, 'Dropout_Model'] = 0.4
commits_df.loc[n, 'CV_logloss'] = 0.014625250378417162
commits_df.loc[n, 'LB_score'] = 0.01844
base_model.save('baseline_model.h5' )<save_to_csv>
n=8
commits_df.loc[n, 'n_commit'] = 14
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.01
commits_df.loc[n, 'train_features'] = 1604
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014713482787703418
commits_df.loc[n, 'LB_score'] = 0.01849
result = base_model.predict(test_x ).clip(0,20)
 result = [each for r in result for each in r]
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 submission = pd.DataFrame({'ID':test.index, 'item_cnt_month':result})
 submission.to_csv('baseline_submission.csv',index =False )<predict_on_test>
n=9
commits_df.loc[n, 'n_commit'] = 18
commits_df.loc[n, 'n_comp_GENES'] = 363
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1219
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014568689235607534
commits_df.loc[n, 'LB_score'] = 0.01841
valid_result = xgb_model.predict(valid_x ).clip(0,20)
 valid_result = [r for r in valid_result] 
<set_options>
n=10
commits_df.loc[n, 'n_commit'] = 19
commits_df.loc[n, 'n_comp_GENES'] = 550
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.91
commits_df.loc[n, 'train_features'] = 1218
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014577509066710863
commits_df.loc[n, 'LB_score'] = 0.01841
random.seed(1234)
 rcParams['figure.figsize'] = 12, 4
warnings.filterwarnings('ignore' )<set_options>
n=11
commits_df.loc[n, 'n_commit'] = 20
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1218
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014572358066092783
commits_df.loc[n, 'LB_score'] = 0.01839
%pwd<load_from_csv>
commits_df['QT_n_quantile_max'] = 100
items = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/items.csv")
 cats = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/item_categories.csv")
 shops = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/shops.csv")
 train = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sales_train.csv")
 test = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv" )<count_missing_values>
n=12
commits_df.loc[n, 'n_commit'] = 21
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1223
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'QT_n_quantile_max'] = 200
commits_df.loc[n, 'CV_logloss'] = 0.014585887029392697
commits_df.loc[n, 'LB_score'] = 0.01841
train.isnull().any()<filter>
n=13
commits_df.loc[n, 'n_commit'] = 22
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1224
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'QT_n_quantile_max'] = 500
commits_df.loc[n, 'CV_logloss'] = 0.014572447523411875
commits_df.loc[n, 'LB_score'] = 0.01840
train = train[(train.item_price < 100000)&(train.item_cnt_day < 1000)]<filter>
n=14
commits_df.loc[n, 'n_commit'] = 23
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1212
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'QT_n_quantile_max'] = 50
commits_df.loc[n, 'CV_logloss'] = 0.014581633680902033
commits_df.loc[n, 'LB_score'] = 0.01840
print(len(train[train.item_price < 0 ]))
print(len(train[train.item_cnt_day < 1 ])) 
<feature_engineering>
commits_df['QT_n_quantile_min'] = commits_df['QT_n_quantile_max']
train = train[train.item_price > 0].reset_index(drop = True)
 train.loc[train.item_cnt_day < 1, "item_cnt_day"] = 0<feature_engineering>
n=15
commits_df.loc[n, 'n_commit'] = 24
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.84
commits_df.loc[n, 'train_features'] = 1215
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'QT_n_quantile_min'] = 10
commits_df.loc[n, 'QT_n_quantile_max'] = 200
commits_df.loc[n, 'CV_logloss'] = 0.014578913453054567
commits_df.loc[n, 'LB_score'] = 0.01840
train.loc[train.shop_id == 0, "shop_id"] = 57
test.loc[test.shop_id == 0 , "shop_id"] = 57
train.loc[train.shop_id == 1, "shop_id"] = 58
test.loc[test.shop_id == 1 , "shop_id"] = 58
train.loc[train.shop_id == 11, "shop_id"] = 10
test.loc[test.shop_id == 11, "shop_id"] = 10
train.loc[train.shop_id == 40, "shop_id"] = 39
test.loc[test.shop_id == 40, "shop_id"] = 39<feature_engineering>
n=15
commits_df.loc[n, 'n_commit'] = 24
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1223
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'QT_n_quantile_min'] = 50
commits_df.loc[n, 'QT_n_quantile_max'] = 1000
commits_df.loc[n, 'CV_logloss'] = 0.014576369890002664
commits_df.loc[n, 'LB_score'] = 0.01842
shops.loc[ shops.shop_name == 'Сергиев Посад ТЦ "7Я"',"shop_name" ] = 'СергиевПосад ТЦ "7Я"'
shops["city"] = shops.shop_name.str.split(" " ).map(lambda x: x[0])
 shops["category"] = shops.shop_name.str.split(" " ).map(lambda x: x[1])
 shops.loc[shops.city == "!Якутск", "city"] = "Якутск" 
<feature_engineering>
commits_df['n_comp_total'] = commits_df['n_comp_GENES'] + commits_df['n_comp_CELLS']
commits_df['seed'] = 42
shops.category = shops.category.apply(lambda x: x if(x in category)else "etc" )<count_unique_values>
commits_df['l_rate'] = 1e-3
commits_df.loc[11, 'l_rate'] = 5e-4
for cat in shops.category.unique() :
print(cat, len(shops[shops.category == cat]))<categorify>
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df = commits_df.sort_values(by=['LB_score', 'CV_logloss'], ascending = True ).reset_index(drop=True)
 commits_df['min'] = 0
commits_df.loc[0, 'min'] = 1
commits_df
shops["shop_category"] = LabelEncoder().fit_transform(shops.category)
 shops["shop_city"] = LabelEncoder().fit_transform(shops.city )<define_variables>
commits_df.sort_values(by=['CV_logloss'], ascending = True )
shops = shops[["shop_id", "shop_category", "shop_city"]]<feature_engineering>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
cats["type_code"] = cats.item_category_name.apply(lambda x: x.split(" ")[0] ).astype(str)
 cats.loc[(cats.type_code == "Игровые")|(cats.type_code == "Аксессуары"), "category" ] = "Игры"<categorify>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
cats.type_code = cats.type_code.apply(lambda x: x if(x in category)else "etc" )<count_unique_values>
for cat in cats.type_code.unique() :
print(cat, len(cats[cats.type_code == cat]))<categorify>
def calc_QT_par_kurt(QT_n_quantile_min=10, QT_n_quantile_max=200):
a = np.array([[1.53655,1], [30.4929,1]])
 b = np.array([QT_n_quantile_min, QT_n_quantile_max])
 return np.linalg.solve(a, b )
cats.type_code = LabelEncoder().fit_transform(cats.type_code)
<feature_engineering>
def n_quantile_for_kurt(kurt, calc_QT_par_kurt_transform):
return int(calc_QT_par_kurt_transform[0]*kurt + calc_QT_par_kurt_transform[1] )
cats["split"] = cats.item_category_name.apply(lambda x: x.split("-"))
cats["subtype"] = cats.split.apply(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats.subtype<categorify>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
cats["subtype_code"] = LabelEncoder().fit_transform(cats["subtype"])
 cats.subtype_code
cats = cats[["item_category_id", "type_code", "subtype_code"]]<categorify>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x<feature_engineering>
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
items["name1"], items["name2"] = items.item_name.str.split("[", 1 ).str
items["name1"], items["name3"] = items.item_name.str.split("(", 1 ).str
items["name2"] = items.name2.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items["name3"] = items.name3.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items = items.fillna('0')
 items["item_name"] = items["item_name"].apply(lambda x: name_correction(x))
items.name2 = items.name2.apply(lambda x: x[:-1] if x !="0" else "0" )<count_values>
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
items.name2.value_counts()<feature_engineering>
data = train_features.append(test_features)
 data
items["type"] = items.name2.apply(lambda x: x[0:8] if x.split(" ")[0] == "xbox" else x.split(" ")[0] )<count_values>
var_thresh = VarianceThreshold(VarianceThreshold_for_FS)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
items.type.value_counts()<feature_engineering>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
items.loc[(items.type == "x360")|(items.type == "xbox360")| 
(items.type == "xbox 360"),"type"] = "xbox 360"
items.loc[ items.type == "", "type"] = "mac"
items.type = items.type.apply(lambda x: x.replace(" ", ""))
items.loc[(items.type == 'pc')|(items.type == 'pс')|(items.type == "pc"), "type" ] = "pc"
items.loc[ items.type == 'рs3' , "type"] = "ps3" 
<groupby>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
group_sum = items.groupby(["type"] ).agg({"item_id": "count"})
 group_sum = group_sum.reset_index()
print(group_sum )<count_values>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
items.name1.value_counts()<count_values>
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds
items.name1.value_counts()<count_values>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
items.name2.value_counts()<count_values>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
items.name3.value_counts()<drop_column>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
items.name2 = items.name2.apply(lambda x: "etc" if(x in drop_cols)else x)
 items = items.drop(["type"], axis = 1 )<categorify>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
items.name2 = LabelEncoder().fit_transform(items.name2)
 items.name3 = LabelEncoder().fit_transform(items.name3)
 items.drop(["item_name", "name1"],axis = 1, inplace= True)
 items.head()<count_unique_values>
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
train['date_block_num'].nunique()<data_type_conversions>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1500
%%time
matrix = []
cols = ["date_block_num", "shop_id", "item_id"]
cols = ["date_block_num", "shop_id", "item_id"]
for i in range(34):
sales = train[train.date_block_num == i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype = np.int16))
matrix = pd.DataFrame(np.vstack(matrix), columns = cols)
 matrix["date_block_num"] = matrix["date_block_num"].astype(np.int8)
 matrix["shop_id"] = matrix["shop_id"].astype(np.int8)
 matrix["item_id"] = matrix["item_id"].astype(np.int16)
 matrix.sort_values(cols, inplace = True )<feature_engineering>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(Dropout_Model)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(Dropout_Model)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
train["revenue"] = train["item_cnt_day"] * train["item_price"]<groupby>
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
group = train.groupby(["date_block_num", "shop_id", "item_id"] ).agg({"item_cnt_day": ["sum"]})
 group.head()<count_missing_values>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
group.isnull().sum()<data_type_conversions>
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
%time
matrix = pd.merge(matrix, group, on = cols, how = "left")
 matrix["item_cnt_month"] = matrix["item_cnt_month"].fillna(0 ).clip(0,20 ).astype(np.float16)
<data_type_conversions>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
test["date_block_num"] = 34
test["date_block_num"] = test["date_block_num"].astype(np.int8)
 test["shop_id"] = test.shop_id.astype(np.int8)
 test["item_id"] = test.item_id.astype(np.int16)
 test.head()<concatenate>
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )