Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<save_to_csv>
sys.path.append('.. /input/iterativestratification')
 warnings.filterwarnings('ignore')
 os.listdir('.. /input/lish-moa')
 pd.set_option('max_columns', 2000 )
test[['ID', 'item_cnt_month']].to_csv("lgbm_submission.csv", index = False )<train_model>
n_comp_GENES = 450
n_comp_CELLS = 2
VarianceThreshold_for_FS = 0.8
Dropout_Model = 0.25
print('n_comp_GENES', n_comp_GENES, 'n_comp_CELLS', n_comp_CELLS, 'total', n_comp_GENES + n_comp_CELLS )
joblib.dump(lgbmr_trained, 'lgbm_model.joblib' )<set_options>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
%matplotlib inline<load_from_csv>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 sales = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 item_cats = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv' )<filter>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=1000,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
train = sales[(sales.item_price < 100000)&(sales.item_price > 0)]
train = train[sales.item_cnt_day < 1001]<feature_engineering>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 40, 'shop_id'] = 39
test.loc[test.shop_id == 40, 'shop_id'] = 39<data_type_conversions>
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(TruncatedSVD(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16)
 df = pd.concat([df, test], ignore_index=True, sort=False, keys=index_cols)
 df.fillna(0, inplace=True )<categorify>
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(TruncatedSVD(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
shops['city'] = shops['shop_name'].apply(lambda x: x.split() [0].lower())
 shops.loc[shops.city == '!якутск', 'city'] = 'якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 coords = dict()
coords['якутск'] =(62.028098, 129.732555, 4)
 coords['адыгея'] =(44.609764, 40.100516, 3)
 coords['балашиха'] =(55.8094500, 37.9580600, 1)
 coords['волжский'] =(53.4305800, 50.1190000, 3)
 coords['вологда'] =(59.2239000, 39.8839800, 2)
 coords['воронеж'] =(51.6720400, 39.1843000, 3)
 coords['выездная'] =(0, 0, 0)
 coords['жуковский'] =(55.5952800, 38.1202800, 1)
 coords['интернет-магазин'] =(0, 0, 0)
 coords['казань'] =(55.7887400, 49.1221400, 4)
 coords['калуга'] =(54.5293000, 36.2754200, 4)
 coords['коломна'] =(55.0794400, 38.7783300, 4)
 coords['красноярск'] =(56.0183900, 92.8671700, 4)
 coords['курск'] =(51.7373300, 36.1873500, 3)
 coords['москва'] =(55.7522200, 37.6155600, 1)
 coords['мытищи'] =(55.9116300, 37.7307600, 1)
 coords['н.новгород'] =(56.3286700, 44.0020500, 4)
 coords['новосибирск'] =(55.0415000, 82.9346000, 4)
 coords['омск'] =(54.9924400, 73.3685900, 4)
 coords['ростовнадону'] =(47.2313500, 39.7232800, 3)
 coords['спб'] =(59.9386300, 30.3141300, 2)
 coords['самара'] =(53.2000700, 50.1500000, 4)
 coords['сергиев'] =(56.3000000, 38.1333300, 4)
 coords['сургут'] =(61.2500000, 73.4166700, 4)
 coords['томск'] =(56.4977100, 84.9743700, 4)
 coords['тюмень'] =(57.1522200, 65.5272200, 4)
 coords['уфа'] =(54.7430600, 55.9677900, 4)
 coords['химки'] =(55.8970400, 37.4296900, 1)
 coords['цифровой'] =(0, 0, 0)
 coords['чехов'] =(55.1477000, 37.4772800, 4)
 coords['ярославль'] =(57.6298700, 39.8736800, 2)
 shops['city_coord_1'] = shops['city'].apply(lambda x: coords[x][0])
 shops['city_coord_2'] = shops['city'].apply(lambda x: coords[x][1])
 shops['country_part'] = shops['city'].apply(lambda x: coords[x][2])
 shops = shops[['shop_id', 'city_code', 'city_coord_1', 'city_coord_2', 'country_part']]<merge>
var_thresh = VarianceThreshold(VarianceThreshold_for_FS)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
df = pd.merge(df, shops, on=['shop_id'], how='left' )<merge>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
map_dict = {
'Чистые носители(штучные)': 'Чистые носители',
'Чистые носители(шпиль)' : 'Чистые носители',
'PC ': 'Аксессуары',
'Служебные': 'Служебные '
}
items = pd.merge(items, item_cats, on='item_category_id')
 items['item_category'] = items['item_category_name'].apply(lambda x: x.split('-')[0])
 items['item_category'] = items['item_category'].apply(lambda x: map_dict[x] if x in map_dict.keys() else x)
 items['item_category_common'] = LabelEncoder().fit_transform(items['item_category'])
 items['item_category_code'] = LabelEncoder().fit_transform(items['item_category_name'])
 items = items[['item_id', 'item_category_common', 'item_category_code']]<merge>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
df = pd.merge(df, items, on=['item_id'], how='left' )<feature_engineering>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
def count_days(date_block_num):
year = 2013 + date_block_num // 12
month = 1 + date_block_num % 12
weeknd_count = len([1 for i in calendar.monthcalendar(year, month)if i[6] != 0])
 days_in_month = calendar.monthrange(year, month)[1]
return weeknd_count, days_in_month, month
map_dict = {i: count_days(i)for i in range(35)}
df['weeknd_count'] = df['date_block_num'].apply(lambda x: map_dict[x][0])
 df['days_in_month'] = df['date_block_num'].apply(lambda x: map_dict[x][1] )<groupby>
tail_cols = tail_cols.index
first_item_block = df.groupby(['item_id'])['date_block_num'].min().reset_index()
first_item_block['item_first_interaction'] = 1
first_shop_item_buy_block = df[df['date_block_num'] > 0].groupby(['shop_id', 'item_id'])['date_block_num'].min().reset_index()
first_shop_item_buy_block['first_date_block_num'] = first_shop_item_buy_block['date_block_num']<merge>
def get_tail_label(df):

 columns = df.columns
n = len(columns)
 irpl = np.zeros(n)
 for column in range(n):
irpl[column] = df[columns[column]].value_counts() [1]
irpl = max(irpl)/irpl
mir = np.average(irpl)
 tail_label = []
for i in range(n):
if irpl[i] > mir:
tail_label.append(columns[i])
 return tail_label
def get_index(df):

 tail_labels = get_tail_label(df)
 index = set()
for tail_label in tail_labels:
sub_index = set(df[df[tail_label]==1].index)
 index = index.union(sub_index)
 return list(index)
 def get_minority_instace(X, y):

 index = get_index(y)
 X_sub = X[X.index.isin(index)].reset_index(drop = True)
 y_sub = y[y.index.isin(index)].reset_index(drop = True)
 return X_sub, y_sub
def nearest_neighbour(X):

 nbs=NearestNeighbors(n_neighbors=120 ,metric='euclidean',algorithm='kd_tree' ).fit(X)
 euclidean,indices= nbs.kneighbors(X)
 return indices
def MLSMOTE(X,y, n_sample):

 indices2 = nearest_neighbour(X)
 n = len(indices2)
 new_X = np.zeros(( n_sample, X.shape[1]))
target = np.zeros(( n_sample, y.shape[1]))
for i in range(n_sample):
reference = random.randint(0,n-1)
 neighbour = random.choice(indices2[reference,1:])
 all_point = indices2[reference]
nn_df = y[y.index.isin(all_point)]
ser = nn_df.sum(axis = 0, skipna = True)
 target[i] = np.array([1 if val>2 else 0 for val in ser])
 ratio = random.random()
gap = X.loc[reference,:] - X.loc[neighbour,:]
new_X[i] = np.array(X.loc[reference,:] + ratio * gap)
 new_X = pd.DataFrame(new_X, columns=X.columns)
 target = pd.DataFrame(target, columns=y.columns)
 new_X = pd.concat([X, new_X], axis=0)
 target = pd.concat([y, target], axis=0)
 return new_X, target
df = pd.merge(df, first_item_block[['item_id', 'date_block_num', 'item_first_interaction']], on=['item_id', 'date_block_num'], how='left')
 df = pd.merge(df, first_shop_item_buy_block[['item_id', 'shop_id', 'first_date_block_num']], on=['item_id', 'shop_id'], how='left')
 df['first_date_block_num'].fillna(100, inplace=True)
 df['shop_item_sold_before'] =(df['first_date_block_num'] < df['date_block_num'] ).astype('int8')
 df.drop(['first_date_block_num'], axis=1, inplace=True)
 df['item_first_interaction'].fillna(0, inplace=True)
 df['shop_item_sold_before'].fillna(0, inplace=True)
 df['item_first_interaction'] = df['item_first_interaction'].astype('int8')
 df['shop_item_sold_before'] = df['shop_item_sold_before'].astype('int8' )<categorify>
train_s = train.drop(['sig_id'] , axis = 1)
 train_s = pd.get_dummies(train_s, columns=['cp_time','cp_dose'])
 X= train_s.drop(target_cols , axis = 1)
 y = train_s[target_cols]
X_sub, y_sub = get_minority_instace(X, y)
 X_res,y_res =MLSMOTE(X_sub, y_sub, 500)
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 df[col+'_lag_'+str(i)] = df[col+'_lag_'+str(i)].astype('float16')
 return df<merge>
X_res.reset_index(drop=True , inplace= True)
 y_res.reset_index(drop=True , inplace= True )
index_cols = ['shop_id', 'item_id', 'date_block_num']
group = train.groupby(index_cols)['item_price'].mean().reset_index().rename(columns={"item_price": "avg_shop_price"}, errors="raise")
 df = pd.merge(df, group, on=index_cols, how='left')
 df['avg_shop_price'] =(df['avg_shop_price'] 
.fillna(0)
.astype(np.float16))
index_cols = ['item_id', 'date_block_num']
group = train.groupby(['date_block_num','item_id'])['item_price'].mean().reset_index().rename(columns={"item_price": "avg_item_price"}, errors="raise")
 df = pd.merge(df, group, on=index_cols, how='left')
 df['avg_item_price'] =(df['avg_item_price'] 
.fillna(0)
.astype(np.float16))
df['item_shop_price_avg'] =(df['avg_shop_price'] - df['avg_item_price'])/ df['avg_item_price']
df['item_shop_price_avg'].fillna(0, inplace=True)
 df = lag_feature(df, [1, 2, 3], 'item_shop_price_avg')
 df.drop(['avg_shop_price', 'avg_item_price', 'item_shop_price_avg'], axis=1, inplace=True )<categorify>
train = train.drop(['sig_id'] , axis =1)
 train = pd.get_dummies(train, columns=['cp_time','cp_dose'] )
item_id_target_mean = df.groupby(['date_block_num','item_id'])['item_cnt_month'].mean().reset_index().rename(columns={"item_cnt_month": "item_target_enc"}, errors="raise")
 df = pd.merge(df, item_id_target_mean, on=['date_block_num','item_id'], how='left')
 df['item_target_enc'] =(df['item_target_enc'] 
.fillna(0)
.astype(np.float16))
df = lag_feature(df, [1, 2, 3], 'item_target_enc')
 df.drop(['item_target_enc'], axis=1, inplace=True )<categorify>
train_res = pd.concat([X_res, y_res] , axis=1 )
item_id_target_mean = df.groupby(['date_block_num','item_id', 'city_code'])['item_cnt_month'].mean().reset_index().rename(columns={
"item_cnt_month": "item_loc_target_enc"}, errors="raise")
 df = pd.merge(df, item_id_target_mean, on=['date_block_num','item_id', 'city_code'], how='left')
 df['item_loc_target_enc'] =(df['item_loc_target_enc'] 
.fillna(0)
.astype(np.float16))
df = lag_feature(df, [1, 2, 3], 'item_loc_target_enc')
 df.drop(['item_loc_target_enc'], axis=1, inplace=True )<categorify>
train_res_f = train.append(train_res )
item_id_target_mean = df.groupby(['date_block_num','item_id', 'shop_id'])['item_cnt_month'].mean().reset_index().rename(columns={
"item_cnt_month": "item_shop_target_enc"}, errors="raise")
 df = pd.merge(df, item_id_target_mean, on=['date_block_num','item_id', 'shop_id'], how='left')
 df['item_shop_target_enc'] =(df['item_shop_target_enc'] 
.fillna(0)
.astype(np.float16))
df = lag_feature(df, [1, 2, 3], 'item_shop_target_enc')
 df.drop(['item_shop_target_enc'], axis=1, inplace=True )<data_type_conversions>
train_res = train.append(train_res)
 train_res = train_res.reset_index(drop=True )
item_id_target_mean = df[df['item_first_interaction'] == 1].groupby(['date_block_num','item_category_code'])['item_cnt_month'].mean().reset_index().rename(columns={
"item_cnt_month": "new_item_cat_avg"}, errors="raise")
 df = pd.merge(df, item_id_target_mean, on=['date_block_num','item_category_code'], how='left')
 df['new_item_cat_avg'] =(df['new_item_cat_avg'] 
.fillna(0)
.astype(np.float16))
df = lag_feature(df, [1, 2, 3], 'new_item_cat_avg')
 df.drop(['new_item_cat_avg'], axis=1, inplace=True )<merge>
folds = train_res.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train_res.drop(target_cols , axis=1), y=train_res[target_cols])) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds
item_id_target_mean = df[df['item_first_interaction'] == 1].groupby(['date_block_num','item_category_code', 'shop_id'])['item_cnt_month'].mean().reset_index().rename(columns={
"item_cnt_month": "new_item_shop_cat_avg"}, errors="raise")
 df = pd.merge(df, item_id_target_mean, on=['date_block_num','item_category_code', 'shop_id'], how='left')
 df['new_item_shop_cat_avg'] =(df['new_item_shop_cat_avg'] 
.fillna(0)
.astype(np.float16))
df = lag_feature(df, [1, 2, 3], 'new_item_shop_cat_avg')
 df.drop(['new_item_shop_cat_avg'], axis=1, inplace=True )<merge>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def lag_feature_adv(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)+'_adv']
shifted['date_block_num'] += i
shifted['item_id'] -= 1
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 df[col+'_lag_'+str(i)+'_adv'] = df[col+'_lag_'+str(i)+'_adv'].astype('float16')
 return df
df = lag_feature_adv(df, [1, 2, 3], 'item_cnt_month' )<correct_missing_values>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
df.fillna(0, inplace=True)
 df = df[(df['date_block_num'] > 2)]
df.head()<load_pretrained>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
df.drop(['ID'], axis=1, inplace=True, errors='ignore')
 df.to_pickle('df.pkl' )<categorify>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
df = pd.read_pickle('df.pkl')
 df.info()<prepare_x_and_y>
feature_cols = [c for c in folds.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
X_train = df[df.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = df[df.date_block_num < 33]['item_cnt_month']
X_valid = df[df.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = df[df.date_block_num == 33]['item_cnt_month']
X_test = df[df.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 del df<init_hyperparams>
test_ = process_data(test )
feature_name = X_train.columns.tolist()
params = {
'objective': 'mse',
'metric': 'rmse',
'num_leaves': 2 ** 7 - 1,
'learning_rate': 0.005,
'feature_fraction': 0.75,
'bagging_fraction': 0.75,
'bagging_freq': 5,
'seed': 1,
'verbose': 1
}
feature_name_indexes = [
'country_part',
'item_category_common',
'item_category_code',
'city_code',
]
lgb_train = lgb.Dataset(X_train[feature_name], Y_train)
 lgb_eval = lgb.Dataset(X_valid[feature_name], Y_valid, reference=lgb_train)
 evals_result = {}
gbm = lgb.train(
 params,
lgb_train,
num_boost_round=3000,
valid_sets=(lgb_train, lgb_eval),
feature_name = feature_name,
categorical_feature = feature_name_indexes,
verbose_eval=5,
evals_result = evals_result,
early_stopping_rounds = 100)
<save_to_csv>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1500
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 Y_test = gbm.predict(X_test[feature_name] ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('gbm_submission.csv', index=False )<load_from_csv>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(Dropout_Model)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(Dropout_Model)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
test_set_full_raw = pd.read_csv(".. /input/sf-crime/test.csv.zip", compression="zip", index_col='Id')
 train_set_full_raw = pd.read_csv(".. /input/sf-crime/train.csv.zip", compression="zip" )<filter>
def run_training(fold, seed):
seed_everything(seed)
 train = folds
test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
ts = train_set_full_raw.copy()
ts = ts[ts["Y"]<90]<count_values>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
ts["Category"].value_counts()<groupby>
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
<feature_engineering><EOS> 
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )