Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<feature_engineering>
sys.path.append('.. /input/iterativestratification')
 warnings.filterwarnings('ignore')
 os.listdir('.. /input/lish-moa')
 pd.set_option('max_columns', 2000 )
def fun_first_item_shop(x):
d={}
d['first_sale_item_shop']=x.loc[x.item_cnt_month>0,'date_block_num'].min()
return pd.Series(d)
 def fun_first_item(x):
d={}
d['first_sale_item']=x.loc[x.item_cnt_month>0,'date_block_num'].min()
return pd.Series(d)
 def first(mat):
group1=mat.groupby(['item_id','shop_id'])[['date_block_num','item_cnt_month','item_id','shop_id']].apply(fun_first_item_shop)
 group2=mat.groupby(['item_id'])[['date_block_num','item_cnt_month','item_id','shop_id']].apply(fun_first_item)
 mat=pd.merge(mat,group1,on=['item_id','shop_id'],how='left')
 mat=pd.merge(mat,group2,on=['item_id'],how='left')
 mat['first_sale_item_shop_period']=mat['date_block_num']-mat['first_sale_item_shop']
mat['first_sale_item_period']=mat['date_block_num']-mat['first_sale_item']
mat.loc[mat.first_sale_item_shop_period<0,'first_sale_item_shop_period']=0
mat.loc[mat.first_sale_item_period<0,'first_sale_item_period']=0
mat.drop(['first_sale_item_shop','first_sale_item'],axis=1,inplace=True)
 return mat<feature_engineering>
n_comp_GENES = 450
n_comp_CELLS = 2
VarianceThreshold_for_FS = 0.9
Dropout_Model = 0.25
QT_n_quantile_min=50,
QT_n_quantile_max=1000,
print('n_comp_GENES', n_comp_GENES, 'n_comp_CELLS', n_comp_CELLS, 'total', n_comp_GENES + n_comp_CELLS )
def fun_last_item_shop(x):
d={}
d['last_sale_item_shop']=x.loc[x.item_cnt_month>0,'date_block_num'].max()
return pd.Series(d)
 def fun_last_item(x):
d={}
d['last_sale_item']=x.loc[x.item_cnt_month>0,'date_block_num'].max()
return pd.Series(d)
 def last(mat):
group1=mat.groupby(['item_id','shop_id'])[['date_block_num','item_cnt_month','item_id','shop_id']].apply(fun_last_item_shop)
 group2=mat.groupby(['item_id'])[['date_block_num','item_cnt_month','item_id','shop_id']].apply(fun_last_item)
 mat=pd.merge(mat,group1,on=['item_id','shop_id'],how='left')
 mat=pd.merge(mat,group2,on=['item_id'],how='left')
 mat['last_sale_item_shop_period']=mat['date_block_num']-mat['last_sale_item_shop']
mat['last_sale_item_period']=mat['date_block_num']-mat['last_sale_item']
mat.loc[mat.last_sale_item_shop_period<0,'last_sale_item_shop_period']=0
mat.loc[mat.last_sale_item_period<0,'last_sale_item_period']=0
mat.drop(['last_sale_item_shop','last_sale_item'],axis=1,inplace=True)
 return mat<sort_values>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_drugs = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
matrix=first(matrix)
 matrix=last(matrix )<define_variables>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
frls_list=['first_sale_item_shop_period','first_sale_item_period','last_sale_item_shop_period','last_sale_item_period']<data_type_conversions>
matrix[frls_list]=matrix[frls_list].fillna(0 )<data_type_conversions>
def calc_QT_par_kurt(QT_n_quantile_min=10, QT_n_quantile_max=200):
a = np.array([[1.53655,1], [30.4929,1]])
 b = np.array([QT_n_quantile_min, QT_n_quantile_max])
 return np.linalg.solve(a, b )
matrix[frls_list]=matrix[frls_list].astype(np.int8 )<feature_engineering>
def n_quantile_for_kurt(kurt, calc_QT_par_kurt_transform):
return int(calc_QT_par_kurt_transform[0]*kurt + calc_QT_par_kurt_transform[1] )
matrix['month'] = matrix['date_block_num'] % 12 
<categorify>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )<load_pretrained>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
matrix.to_pickle('data.pkl')
<load_pretrained>
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(TruncatedSVD(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
data = pd.read_pickle('data.pkl' )<filter>
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(TruncatedSVD(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
data = data[data.date_block_num > 11]<drop_column>
data = train_features.append(test_features)
 data
data=data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag1',
'item_cnt_month_lag2',
'item_cnt_month_lag3',
'item_cnt_month_lag6',
'item_cnt_month_lag12',
'date_avg_item_cnt_lag1',
'date_item_avg_item_cnt_lag1',
'date_item_avg_item_cnt_lag2',
'date_item_avg_item_cnt_lag3',
'date_item_avg_item_cnt_lag6',
'date_item_avg_item_cnt_lag12',
'date_shop_avg_item_cnt_lag1',
'date_shop_avg_item_cnt_lag2',
'date_shop_avg_item_cnt_lag3',
'date_shop_avg_item_cnt_lag6',
'date_shop_avg_item_cnt_lag12',
'date_cat_avg_item_cnt_lag1',
'date_city_avg_item_cnt_lag1',
'date_shop_cat_avg_item_cnt_lag1',
'date_item_city_avg_item_cnt_lag1',
'delta_price_lag',
'first_sale_item_shop_period',
'first_sale_item_period',
'month'
]]<prepare_x_and_y>
var_thresh = VarianceThreshold(VarianceThreshold_for_FS)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<choose_model_class>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
model = XGBRegressor(
 tree_method='gpu_hist',
max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
random_state=42 )<train_model>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
model.fit(
 X_train,
Y_train,
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
eval_metric="rmse",
verbose=True,
early_stopping_rounds = 10 )<save_to_csv>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False )<define_variables>
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds
%matplotlib inline
sales_train = '.. /input/competitive-data-science-predict-future-sales/sales_train.csv'
test = '.. /input/competitive-data-science-predict-future-sales/test.csv'
sample_submission = '.. /input/competitive-data-science-predict-future-sales/sample_submission.csv'
item_categories = '.. /input/competitive-data-science-predict-future-sales/item_categories.csv'
items = '.. /input/competitive-data-science-predict-future-sales/items.csv'
shops = '.. /input/competitive-data-science-predict-future-sales/shops.csv'
train_data = pd.read_csv(sales_train)
 test_data = pd.read_csv(test)
 items_data = pd.read_csv(items)
 shops_data = pd.read_csv(shops)
 item_cat_data = pd.read_csv(item_categories)
 submission_file = pd.read_csv(sample_submission)
 test_data = test_data.drop('ID', axis = 1)
 print(train_data.shape )<count_duplicates>
max(train_drugs['sig_id'].value_counts().values)
train_data = train_data [train_data.duplicated() == False]
train_data.shape<data_type_conversions>
merged_drug = folds.merge(train_drugs)
def downcast_dtypes(df):
float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype in ["int64", "int32"]]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df
train_data = downcast_dtypes(train_data)
 test_data = downcast_dtypes(test_data)
 items_data = downcast_dtypes(items_data)
 shops_data = downcast_dtypes(shops_data)
 item_cat_data = downcast_dtypes(item_cat_data )<define_variables>
merged_drug['drug_id'].value_counts()
col_dsi = ['date_block_num', 'shop_id', 'item_id']
col_si = ['shop_id', 'item_id']
col_di = ['date_block_num', 'item_id']
col_ds = ['date_block_num', 'shop_id']
col_d = ['date_block_num']
col_s = ['shop_id']
col_i = ['item_id']
col_dsicat = ['date_block_num', 'shop_id', 'item_category_id']
col_dicati = ['date_block_num', 'item_category_id', 'item_id']
col_dp = ['date_block_num', 'price_bins']<groupby>
merge_grouped = merged_drug.groupby(by = 'drug_id')[target_cols].sum()
top_itemscat = list(items_data.groupby('item_category_id')['item_id'].count().nlargest(15 ).index )<define_variables>
drug_ids = list(merge_grouped[merge_grouped['sum_tail_enders'] >0].index)
l_cat = list(item_cat_data.item_category_name)
 for ind in range(0,1):
l_cat[ind] = 'PC Headsets / Headphones'
for ind in range(1,8):
l_cat[ind] = 'Access'
l_cat[8] = 'Tickets(figure)'
l_cat[9] = 'Delivery of goods'
for ind in range(10,18):
l_cat[ind] = 'Consoles'
for ind in range(18,25):
l_cat[ind] = 'Consoles Games'
l_cat[25] = 'Accessories for games'
for ind in range(26,28):
l_cat[ind] = 'phone games'
for ind in range(28,32):
l_cat[ind] = 'CD games'
for ind in range(32,37):
l_cat[ind] = 'Card'
for ind in range(37,43):
l_cat[ind] = 'Movie'
for ind in range(43,55):
l_cat[ind] = 'Books'
for ind in range(55,61):
l_cat[ind] = 'Music'
for ind in range(61,73):
l_cat[ind] = 'Gifts'
for ind in range(73,79):
l_cat[ind] = 'Soft'
for ind in range(79,81):
l_cat[ind] = 'Office'
for ind in range(81,83):
l_cat[ind] = 'Clean'
l_cat[83] = 'Elements of a food'
item_cat_data ['item_category_name'] = pd.DataFrame(l_cat)
 items_data = items_data.merge(item_cat_data, on = 'item_category_id', how = 'left' ).drop(['item_name'], axis =1 )<drop_column>
x = merged_drug['drug_id'].value_counts()
shops_data ['city'] = shops_data ['shop_name'].apply(lambda x : x.split(' ')[0])
 shops_data ['center'] = shops_data ['shop_name'].apply(lambda x : x.split(' ')[1])
 shops_data = shops_data.drop('shop_name', axis = 1 )<categorify>
all_drugs = merged_drug['drug_id'].value_counts()
items_data ['item_category_name'] = LabelEncoder().fit_transform(items_data ['item_category_name'] ).astype(np.int8 )<data_type_conversions>
samp_strat = {}
shops_data ['city'] = LabelEncoder().fit_transform(shops_data ['city'] ).astype(np.int8)
 shops_data ['center'] = LabelEncoder().fit_transform(shops_data ['center'] ).astype(np.int8 )<data_type_conversions>
samp_strat = {}
for i in list(all_drugs.index):
if i in drug_ids:
samp_strat.update({i:150})
 else:
if(( all_drugs[i]+1)< 7):
samp_strat.update({i:7})
 else:
samp_strat.update({i:all_drugs[i]+1})
train_data ['date'] = pd.to_datetime(train_data ['date'], format = '%d.%m.%Y')
 train_data ['item_price'] = round(train_data ['item_price'],1 )<categorify>
drug_smote = pd.get_dummies(merged_drug, columns=['cp_time','cp_dose'] )
rus_holidays = holidays.Russia()
train_data ['is_holiday'] = train_data ['date'].apply(lambda x : 1 if x in rus_holidays else 0)
 holiday_month = train_data [train_data.is_holiday == 1].groupby('date_block_num')['date'].nunique()
holidays_df = pd.DataFrame(holiday_month ).reset_index()
holidays_df.columns = ['date_block_num', 'holiday_count']
master = pd.DataFrame(index = list(range(0,35)) , columns = ['holidays'] ).reset_index()
master.columns = ['date_block_num', 'holidays']
master ['holidays'] = 0
holidays = master.merge(holidays_df, on = 'date_block_num', how = 'left' ).fillna(0 ).drop('holidays', axis = 1)
 holidays ['date_block_num'] = holidays ['date_block_num'].astype(np.int8)
 holidays ['holiday_count'] = holidays ['holiday_count'].astype(np.int8 )<filter>
y_smote = merged_drug['drug_id']
x_smote = drug_smote.drop(target_cols+['sig_id' , 'kfold', 'drug_id'] , axis =1 )
train_data = train_data.loc [(train_data ['item_price'] >= 0)&(train_data ['item_price'] < 300000)&(train_data ['item_cnt_day'] >=0)&(train_data ['item_cnt_day'] < 2000), :]
train_data = train_data [['date_block_num', 'shop_id', 'item_id', 'item_price', 'item_cnt_day']]<filter>
smote = RandomOverSampler(sampling_strategy= samp_strat , random_state=42)
td_piv = train_data.pivot_table(index = 'item_id', columns = ['date_block_num'], values = 'item_cnt_day', aggfunc = 'sum' ).fillna(0)
 last6 = td_piv.loc [:,td_piv.columns [27:33]]
inactive_items = list(last6[last6.iloc [:, :].sum(axis = 1)== 0].index)
 active_items = list(td_piv [td_piv.index.isin(inactive_items)== False].index)
 print('active items {}'.format(len(active_items)))
 print('Inactive items {}'.format(len(inactive_items)))
 inactive_test_items = list(set(test_data.loc [test_data ['item_id'].isin(inactive_items), 'item_id'].values))
print('inactive_test_items {}'.format(len(inactive_test_items)))
 active_test_items = list(set(test_data.loc [test_data ['item_id'].isin(inactive_test_items)== False, 'item_id'].values))
print('active_test_items {}'.format(len(active_test_items)))
 new_items = list(set(test_data.item_id)-set(train_data.item_id))
no_forecast_items = inactive_test_items + new_items
print('no_forecast_items : {}'.format(len(no_forecast_items)) )<feature_engineering>
X_res, y_res = smote.fit_resample(x_smote, y_smote )
train_data ['revenue'] = train_data ['item_price'] * train_data ['item_cnt_day']<merge>
merged_drug_to_targets = merged_drug.groupby(by = 'drug_id')[target_cols].max()
revenue = train_data.groupby(col_dsi)['revenue'].sum().reset_index()
revenue.columns = col_dsi + ['revenue']
target = train_data.groupby(col_dsi)['item_cnt_day'].sum().reset_index()
target.columns = col_dsi + ['target']
price = train_data.groupby(col_dsi ).agg({'item_price' : ['mean']} ).reset_index().fillna(0)
 price.columns = col_dsi + ['price']
train_data = train_data [col_dsi].merge(target, on = col_dsi, how = 'left' ).fillna(0)
 train_data ['target'] = train_data ['target'].astype(np.float16)
 train_data = train_data.merge(price, on = col_dsi, how = 'left' ).fillna(0)
 train_data = train_data.merge(revenue, on = col_dsi, how = 'left' ).fillna(0)
 train_data = downcast_dtypes(train_data )<data_type_conversions>
train_smote = pd.concat([X_res, y_res], axis=1 )
train_data ['date_block_num'] = train_data ['date_block_num'].astype(np.int8)
 train_data ['shop_id'] = train_data ['shop_id'].astype(np.int8)
 train_data ['price'] = train_data ['price'].astype(np.float16 )<data_type_conversions>
train_smote_f= train_smote.merge(merged_drug_to_targets , on = 'drug_id' )
grid = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train_data[train_data.date_block_num==i]
grid.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
grid = pd.DataFrame(np.vstack(grid), columns=cols)
 grid ['date_block_num'] = grid ['date_block_num'].astype(np.int8)
 grid ['shop_id'] = grid ['shop_id'].astype(np.int8)
 grid ['item_id'] = grid ['item_id'].astype(np.int16)
 grid.sort_values(cols,inplace=True)
<merge>
train_smote_f = train_smote_f.drop(['drug_id'], axis =1 )
train_data = grid.merge(train_data, on = col_dsi, how = 'left' ).drop_duplicates(subset = col_dsi, keep = 'first' ).fillna(0)
 train_data = train_data.sort_values(by = col_dsi, ascending = True )<data_type_conversions>
train_smote_f = train_smote_f.fillna(0)
 folds = train_smote_f.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train_smote_f.drop(target_cols, axis=1), y= train_smote_f[target_cols])) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds
test_data ['date_block_num'] = 34
test_data ['date_block_num'] = test_data ['date_block_num'].astype(np.int8)
 test_data ['shop_id'] = test_data ['shop_id'].astype(np.int8)
 test_data ['item_id'] = test_data ['item_id'].astype(np.int16 )<concatenate>
train = train_smote_f
data = pd.concat([train_data, test_data], axis = 0 ).fillna(0 )<merge>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def lags(train, lags, col):
col_dsi = ['date_block_num', 'shop_id', 'item_id']
df = train
for i in lags:
prev = train [col_dsi + [col]].copy()
prev ['date_block_num'] = prev ['date_block_num'] + i
prev = prev.rename(columns = {col : 'prev_' + col + str(i)})
 df = df.merge(prev, on = col_dsi, how = 'left')
 return df.fillna(0 )<categorify>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
lags_ = [1,2,3,12]
data = lags(data, lags_, 'target' )<drop_column>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
lags_ = [1]
data = lags(data, lags_, 'price' )<merge>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
target_mean = data.groupby('item_id', as_index = False)['price'].max()
target_mean.columns = ['item_id'] + ['item_pricemax']
data = pd.merge(data, target_mean, on = ['item_id'], how = 'left')
 target_mean = data.groupby('item_id', as_index = False)['price'].min()
target_mean.columns = ['item_id'] + ['item_pricemin']
data = pd.merge(data, target_mean, on = ['item_id'], how = 'left')
 data ['discount'] = data ['item_pricemax'] - data ['item_pricemin']
lags_ = [1]
data = lags(data, lags_, 'discount')
 data = data.drop('discount', axis = 1)
 data = data.drop(['item_pricemax','item_pricemin'], axis = 1 )<drop_column>
feature_cols = [c for c in folds.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
lags_ = [1]
data = lags(data, lags_, 'revenue' )<data_type_conversions>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 10
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1500
data ['revenue'] = data ['revenue'].astype(np.float32)
 data ['prev_revenue1'] = data ['prev_revenue1'].astype(np.float32 )<merge>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(Dropout_Model)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(Dropout_Model)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
target_mean = data.groupby(col_di, as_index = False)['target'].mean()
target_mean.columns = col_di + ['target_meandi']
data = pd.merge(data, target_mean, on = col_di, how = 'left')
 lags_ = [1]
data = lags(data, lags_, 'target_meandi')
 data = data.drop('target_meandi', axis = 1 )<merge>
def run_training(fold, seed):
seed_everything(seed)
 train = folds
test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
target_mean = data.groupby(col_ds, as_index = False)['target'].mean()
target_mean.columns = col_ds + ['target_meands']
data = pd.merge(data, target_mean, on = col_ds, how = 'left')
 lags_ = [1]
data = lags(data, lags_, 'target_meands')
 data = data.drop('target_meands', axis = 1 )<merge>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
target_mean = data.groupby(col_ds, as_index = False)['revenue'].mean()
target_mean.columns = col_ds + ['revenue_means']
data = pd.merge(data, target_mean, on = col_ds, how = 'left')
 lags_ = [1]
data = lags(data, lags_, 'revenue_means')
 data = data.drop('revenue_means', axis = 1)
 target_mean = data.groupby(col_s, as_index = False)['revenue'].mean()
target_mean.columns = col_s + ['revenue_means_tot']
data = pd.merge(data, target_mean, on = col_s, how = 'left')
 lags_ = [1]
data = lags(data, lags_, 'revenue_means_tot')
 data = data.drop('revenue_means_tot', axis = 1)
<drop_column>
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
<merge><EOS> 
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )