Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<categorify>
%matplotlib inline
warnings.filterwarnings("ignore")
 print(os.listdir(".. /input"))
random_seed = 0
def rmse(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
result = pd.DataFrame({"id": [], "type": []})
 result.id = np.array(np.array(test_ids))
result.type = creature_encoder.inverse_transform(prediction_class )<save_to_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.index = train['id']
test.index = test['id']
result.to_csv("nn_result.csv", index= False )<load_from_csv>
train.isna().sum().sort_values(ascending=False )
df = dtable.fread('/kaggle/input/jane-street-market-prediction/train.csv' ).to_pandas()
df = df.query('date > 85' ).reset_index(drop = True)
 df = df[df.weight > 0]
df.reset_index(inplace=True, drop=True)
 df = df.astype({c: np.float32 for c in df.select_dtypes(include='float64' ).columns})
 df_labels = df[['date', 'weight', 'resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']]
features = [c for c in df.columns if 'feature' in c]
df = df.drop(df_labels.columns, axis=1)
 df.fillna(df.mean() , inplace=True)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']
X = df[features]
y = np.stack([(df_labels[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(df[features[1:]].values, axis=0)
 device = 'cuda' if t.cuda.is_available() else 'cpu'<categorify>
train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1007,'budget'] = 2
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1570,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 1885,'budget'] = 12
train.loc[train['id'] == 2091,'budget'] = 10
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'budget'] = 6
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
train.loc[train['id'] == 335,'budget'] = 2
train.loc[train['id'] == 348,'budget'] = 12
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 1100000
train.loc[train['id'] == 640,'budget'] = 6
train.loc[train['id'] == 696,'budget'] = 1
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 850,'budget'] = 1500000
train.loc[train['id'] == 1199,'budget'] = 5
train.loc[train['id'] == 1282,'budget'] = 9
train.loc[train['id'] == 1347,'budget'] = 1
train.loc[train['id'] == 1755,'budget'] = 2
train.loc[train['id'] == 1801,'budget'] = 5
train.loc[train['id'] == 1918,'budget'] = 592
train.loc[train['id'] == 2033,'budget'] = 4
train.loc[train['id'] == 2118,'budget'] = 344
train.loc[train['id'] == 2252,'budget'] = 130
train.loc[train['id'] == 2256,'budget'] = 1
train.loc[train['id'] == 2696,'budget'] = 10000000
encoder_out_dim=64<categorify>
test.loc[test['id'] == 3033,'budget'] = 250
test.loc[test['id'] == 3051,'budget'] = 50
test.loc[test['id'] == 3084,'budget'] = 337
test.loc[test['id'] == 3224,'budget'] = 4
test.loc[test['id'] == 3594,'budget'] = 25
test.loc[test['id'] == 3619,'budget'] = 500
test.loc[test['id'] == 3831,'budget'] = 3
test.loc[test['id'] == 3935,'budget'] = 500
test.loc[test['id'] == 4049,'budget'] = 995946
test.loc[test['id'] == 4424,'budget'] = 3
test.loc[test['id'] == 4460,'budget'] = 8
test.loc[test['id'] == 4555,'budget'] = 1200000
test.loc[test['id'] == 4624,'budget'] = 30
test.loc[test['id'] == 4645,'budget'] = 500
test.loc[test['id'] == 4709,'budget'] = 450
test.loc[test['id'] == 4839,'budget'] = 7
test.loc[test['id'] == 3125,'budget'] = 25
test.loc[test['id'] == 3142,'budget'] = 1
test.loc[test['id'] == 3201,'budget'] = 450
test.loc[test['id'] == 3222,'budget'] = 6
test.loc[test['id'] == 3545,'budget'] = 38
test.loc[test['id'] == 3670,'budget'] = 18
test.loc[test['id'] == 3792,'budget'] = 19
test.loc[test['id'] == 3881,'budget'] = 7
test.loc[test['id'] == 3969,'budget'] = 400
test.loc[test['id'] == 4196,'budget'] = 6
test.loc[test['id'] == 4221,'budget'] = 11
test.loc[test['id'] == 4222,'budget'] = 500
test.loc[test['id'] == 4285,'budget'] = 11
test.loc[test['id'] == 4319,'budget'] = 1
test.loc[test['id'] == 4639,'budget'] = 10
test.loc[test['id'] == 4719,'budget'] = 45
test.loc[test['id'] == 4822,'budget'] = 22
test.loc[test['id'] == 4829,'budget'] = 20
test.loc[test['id'] == 4969,'budget'] = 20
test.loc[test['id'] == 5021,'budget'] = 40
test.loc[test['id'] == 5035,'budget'] = 1
test.loc[test['id'] == 5063,'budget'] = 14
test.loc[test['id'] == 5119,'budget'] = 2
test.loc[test['id'] == 5214,'budget'] = 30
test.loc[test['id'] == 5221,'budget'] = 50
test.loc[test['id'] == 4903,'budget'] = 15
test.loc[test['id'] == 4983,'budget'] = 3
test.loc[test['id'] == 5102,'budget'] = 28
test.loc[test['id'] == 5217,'budget'] = 75
test.loc[test['id'] == 5224,'budget'] = 3
test.loc[test['id'] == 5469,'budget'] = 20
test.loc[test['id'] == 5840,'budget'] = 1
test.loc[test['id'] == 5960,'budget'] = 30
test.loc[test['id'] == 6506,'budget'] = 11
test.loc[test['id'] == 6553,'budget'] = 280
test.loc[test['id'] == 6561,'budget'] = 7
test.loc[test['id'] == 6582,'budget'] = 218
test.loc[test['id'] == 6638,'budget'] = 5
test.loc[test['id'] == 6749,'budget'] = 8
test.loc[test['id'] == 6759,'budget'] = 50
test.loc[test['id'] == 6856,'budget'] = 10
test.loc[test['id'] == 6858,'budget'] = 100
test.loc[test['id'] == 6876,'budget'] = 250
test.loc[test['id'] == 6972,'budget'] = 1
test.loc[test['id'] == 7079,'budget'] = 8000000
test.loc[test['id'] == 7150,'budget'] = 118
test.loc[test['id'] == 6506,'budget'] = 118
test.loc[test['id'] == 7225,'budget'] = 6
test.loc[test['id'] == 7231,'budget'] = 85
test.loc[test['id'] == 5222,'budget'] = 5
test.loc[test['id'] == 5322,'budget'] = 90
test.loc[test['id'] == 5350,'budget'] = 70
test.loc[test['id'] == 5378,'budget'] = 10
test.loc[test['id'] == 5545,'budget'] = 80
test.loc[test['id'] == 5810,'budget'] = 8
test.loc[test['id'] == 5926,'budget'] = 300
test.loc[test['id'] == 5927,'budget'] = 4
test.loc[test['id'] == 5986,'budget'] = 1
test.loc[test['id'] == 6053,'budget'] = 20
test.loc[test['id'] == 6104,'budget'] = 1
test.loc[test['id'] == 6130,'budget'] = 30
test.loc[test['id'] == 6301,'budget'] = 150
test.loc[test['id'] == 6276,'budget'] = 100
test.loc[test['id'] == 6473,'budget'] = 100
test.loc[test['id'] == 6842,'budget'] = 30
class Autoencoder(nn.Module):
def __init__(self):
super(Autoencoder, self ).__init__()
self.encoder = nn.Sequential(nn.BatchNorm1d(130),
LinBnDrop(130, 256, bn=True, p=0, act=Mish() , lin_first=False),
LinBnDrop(256, 512, bn=True, p=0.12, act=Mish() , lin_first=False),
LinBnDrop(512, 256, bn=True, p=0.12, act=Mish() , lin_first=False),
LinBnDrop(256, encoder_out_dim, bn=False, act=None, lin_first=False))
self.decoder = nn.Sequential(nn.BatchNorm1d(64),
LinBnDrop(64, 400, bn=True, p=0, act=Mish() , lin_first=False),
LinBnDrop(400, 800, bn=True, p=0.12, act=Mish() , lin_first=False),
LinBnDrop(800, 400, bn=True, p=0.12, act=Mish() , lin_first=False),
LinBnDrop(400, 130, bn=False, act=None, lin_first=False))
def encode(self, x):
return self.encoder(x)
 def decode(self, x):
return self.decoder(x)
 def forward(self, x):
encoded = self.encoder(x)
 decoded = self.decoder(encoded)
 return decoded<load_pretrained>
le = LabelEncoder()
ae = Learner(None, Autoencoder() , loss_func=1)
 ae.load('/kaggle/input/jstreetae1/autoencoder_rmse' )<categorify>
dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
dfx = text_to_dict(train)
 for col in dict_columns:
train[col]=dfx[col]
class Model(nn.Module):
def __init__(self, input_dim):
super(Model, self ).__init__()
self.encoder = ae.model.encoder
self.encoder.eval()
self.layers = nn.Sequential(nn.BatchNorm1d(input_dim),
LinBnDrop(input_dim, 400, bn=True, p=0, act=Mish() , lin_first=False),
LinBnDrop(400, 800, bn=True, p=0.2289, act=Mish() , lin_first=False),
LinBnDrop(800, 400, bn=True, p=0.2289, act=Mish() , lin_first=False),
LinBnDrop(400, 5, bn=False, act=None, lin_first=False))
def forward(self, x):
encoded_x = self.encoder(x ).detach()
x = t.cat(( encoded_x, x), dim=1)
 x = self.layers(x)
 return x 
<load_pretrained>
dfx = text_to_dict(test)
 for col in dict_columns:
test[col]=dfx[col]
model_nn = Model(len(features)+ encoder_out_dim)
 model_nn = model_nn.to(device)
 learn = Learner(None, model_nn, loss_func=1)
 learn.load('/kaggle/input/jstreetmlpae1/ae_roc' )<categorify>
def date(x):
x=str(x)
 year=x.split('/')[2]
if int(year)<19:
return x[:-2]+'20'+year
else:
return x[:-2]+'19'+year
train['release_date'] =train['release_date'].fillna('1/1/90' ).apply(lambda x: date(x))
test['release_date'] =test['release_date'].fillna('1/1/90' ).apply(lambda x: date(x))
train['release_date'] = train['release_date'].apply(lambda x: datetime.strptime(x,'%m/%d/%Y'))
test['release_date'] = test['release_date'].apply(lambda x: datetime.strptime(x,'%m/%d/%Y'))
train['release_day'] = train['release_date'].apply(lambda x:x.weekday())
 train['release_month'] = train['release_date'].apply(lambda x:x.month)
 train['release_year'] = train['release_date'].apply(lambda x:x.year)
 test['release_day'] = test['release_date'].apply(lambda x:x.weekday())
 test['release_month'] = test['release_date'].apply(lambda x:x.month)
 test['release_year'] = test['release_date'].apply(lambda x:x.year)
 day=train['release_day'].value_counts().sort_index()
sns.barplot(day.index,day)
 plt.gca().set_xticklabels(["Monday","Tuesday","Wednesday","Thursday","Friday","Saturday","Sunday"],rotation='45')
 plt.ylabel('No of releases' )
@njit
def fillna_npwhere_njit(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array
def for_loop(method, matrix, values):
for i in range(matrix.shape[0]):
matrix[i] = method(matrix[i], values)
 return matrix<find_best_params>
def data_preprocessing(df):
df['belongs_to_collection']= df['belongs_to_collection'].apply(lambda x: 1 if x != {} else 0)
 df['has_homepage'] = 1
df.loc[ pd.isnull(df['homepage']),'has_homepage'] = 0
df['num_genres'] = df['genres'].apply(lambda x: len(x)if x != {} else 0)
 df['budget'] = np.log1p(df['budget'])
 df['isTaglineNA'] = df['tagline'].apply(lambda x: 0 if pd.isnull(x)else 1)
 df['original_language'] = le.fit_transform(df['original_language'])
 df['isOriginalLanguageEng'] = 0
df.loc[ df['original_language'].astype(str)== "en" ,"isOriginalLanguageEng"] = 1
df['ismovie_released']=1
df.loc[(df['status']!='Released'),'ismovie_released']=0
df['no_spoken_languages']=df['spoken_languages'].apply(lambda x: len(x))
df['original_title_letter_count'] = df['original_title'].str.len()
df['original_title_word_count'] = df['original_title'].str.split().str.len()
df['title_word_count'] = df['title'].str.split().str.len()
df['overview_word_count'] = df['overview'].str.split().str.len()
df['tagline_word_count'] = df['tagline'].str.split().str.len()
df['production_countries_count'] = df['production_countries'].apply(lambda x : len(x))
df['production_companies_count'] = df['production_companies'].apply(lambda x : len(x))
df['cast_count'] = df['cast'].apply(lambda x : len(x))
df['crew_count'] = df['crew'].apply(lambda x : len(x))
df['genders_0_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
df['genders_1_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
df['genders_2_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
cols_to_normalize=['runtime','popularity','budget','_budget_runtime_ratio','_budget_year_ratio','_budget_popularity_ratio','_releaseYear_popularity_ratio',
'_releaseYear_popularity_ratio2','_num_Keywords','_num_cast','no_spoken_languages','original_title_letter_count','original_title_word_count',
'title_word_count','overview_word_count','tagline_word_count','production_countries_count','production_companies_count','cast_count','crew_count',
'genders_0_crew','genders_1_crew','genders_2_crew']
for col in cols_to_normalize:
if(col in df):
print(col)
 x_array=[]
x_array=np.array(df[col].fillna(0))
X_norm=normalize([x_array])[0]
df[col]=X_norm
df = df.drop(['homepage','tagline','poster_path','overview','genres',
'imdb_id','original_title','cast','Keywords','title',
'production_companies','production_countries',
'release_date','spoken_languages','crew','status'],axis=1)
 df.fillna(-1,inplace=True)
 return df
%%time
%%capture
env = janestreet.make_env()
learn.model.eval()
preds = []
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
x_tt[:, 1:] = for_loop(fillna_npwhere_njit, x_tt[:, 1:], f_mean)
 pred = 0.
 pred = learn.model(t.from_numpy(x_tt ).to(device, t.float)).sigmoid() [0][-1].item()
preds.append(pred)
 action = 1 if pred >=.5 else 0
pred_df.action = action
else:
pred_df.action = 0
env.predict(pred_df )<compute_test_metric>
preds = np.array(preds)
 preds.mean() , preds.std() , sum(preds >=.5), sum(preds < 5 )<import_modules>
test['revenue'] = np.nan
all_data=data_preprocessing(( pd.concat([train,test])) ).reset_index(drop=True )
warnings.filterwarnings("ignore" )<set_options>
train=all_data.loc[:train.shape[0]-1,:]
test=all_data.loc[train.shape[0]:,:]
print("Train shape",train.shape)
 print("Test shape",train.shape )
DEVICE = torch.device("cuda:0")if torch.cuda.is_available() else torch.device("cpu" )<load_from_csv>
features = list(train.columns)
 features = [i for i in features if i != 'id' and i != 'revenue']
def load_preprocess_data(filename='.. /input/jane-street-market-prediction/train.csv', isTrainData=True):
dtype = None
if isTrainData:
dtype = {
'date' : 'int64',
'weight' : 'float64',
'resp' : 'float64',
'ts_id' : 'int64',
'feature_0' : 'float64'
}
else:
dtype = {
'date' : 'int64',
'weight' : 'float64',
'feature_0' : 'float64'
}
for i in range(1, 130):
k = 'feature_' + str(i)
 dtype[k] = 'float32'
X = pd.read_csv(filename, dtype=dtype)
 resp_cols = ['resp_1', 'resp_2', 'resp_3','resp_4', 'resp']
X = X.query('date > 85')
 X = X[X['weight'] != 0].reset_index(drop = True)
 y = np.stack([(X[c] > 0 ).astype('int')for c in resp_cols] ).T
f_columns = [c for c in X.columns if "feature" in c]
if isTrainData:
X.drop(columns=['weight', 'date', 'resp', 'resp_1', 'resp_2', 'resp_3', 'resp_4', 'ts_id'], inplace=True)
 else:
X.drop(columns=['weight', 'date'], inplace=True)
 preprocess_pipe = Pipeline([ 
("imputer", SimpleImputer(missing_values=np.nan, strategy='mean')) ,
])
 X = preprocess_pipe.fit_transform(X)
 gc.collect()
return X, y, preprocess_pipe<prepare_x_and_y>
def score(data, y):
validation_res = pd.DataFrame(
 {"id": data["id"].values,
"transactionrevenue": data["revenue"].values,
"predictedrevenue": np.expm1(y)})
 validation_res = validation_res.groupby("id")["transactionrevenue", "predictedrevenue"].sum().reset_index()
return np.sqrt(mean_squared_error(np.log1p(validation_res["transactionrevenue"].values),
np.log1p(validation_res["predictedrevenue"].values)) )
X, y, preprocess_pipe = load_preprocess_data()
X.shape<load_from_csv>
class KFoldValidation() :
def __init__(self, data, n_splits=5):
unique_vis = np.array(sorted(data['id'].astype(str ).unique()))
folds = GroupKFold(n_splits)
 ids = np.arange(data.shape[0])
 self.fold_ids = []
for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):
self.fold_ids.append([
ids[data['id'].astype(str ).isin(unique_vis[trn_vis])],
ids[data['id'].astype(str ).isin(unique_vis[val_vis])]
])
 def validate(self, train, test, features, model, name="", prepare_stacking=False,
fit_params={"early_stopping_rounds": 500, "verbose": 100, "eval_metric": "rmse"}):
model.FI = pd.DataFrame(index=features)
 full_score = 0
if prepare_stacking:
test[name] = 0
train[name] = np.NaN
for fold_id,(trn, val)in enumerate(self.fold_ids):
devel = train[features].iloc[trn]
y_devel = np.log1p(train["revenue"].iloc[trn])
 valid = train[features].iloc[val]
y_valid = np.log1p(train["revenue"].iloc[val])
 print("Fold ", fold_id, ":")
 model.fit(devel, y_devel, eval_set=[(valid, y_valid)], **fit_params)
 if len(model.feature_importances_)== len(features):
model.FI['fold' + str(fold_id)] = model.feature_importances_ / model.feature_importances_.sum()
predictions = model.predict(valid)
 predictions[predictions < 0] = 0
print("Fold ", fold_id, " error: ", mean_squared_error(y_valid, predictions)**0.5)
 fold_score = score(train.iloc[val], predictions)
 full_score += fold_score / len(self.fold_ids)
 print("Fold ", fold_id, " score: ", fold_score)
 if prepare_stacking:
train[name].iloc[val] = predictions
test_predictions = model.predict(test[features])
 test_predictions[test_predictions < 0] = 0
test[name] += test_predictions / len(self.fold_ids)
 print("Final score: ", full_score)
 return full_score
dtype = {
'feature' : 'str',
'tag_0' : 'int8'
}
for i in range(1, 29):
k = 'tag_' + str(i)
 dtype[k] = 'int8'
features_df = pd.read_csv('.. /input/jane-street-market-prediction/features.csv', usecols=range(1,30), dtype=dtype)
 N_FEATURES = features_df.shape[0]
N_FEAT_TAGS = features_df.shape[1]
features_df.head()<set_options>
Kfolder = KFoldValidation(train )
del features_df
gc.collect()
N_FEATURES, N_FEAT_TAGS<compute_train_metric>
lgbmodel = lgb.LGBMRegressor(n_estimators=10000,
objective='regression',
metric='rmse',
max_depth = 5,
num_leaves=30,
min_child_samples=100,
learning_rate=0.01,
boosting = 'gbdt',
min_data_in_leaf= 10,
feature_fraction = 0.9,
bagging_freq = 1,
bagging_fraction = 0.9,
importance_type='gain',
lambda_l1 = 0.2,
bagging_seed=random_seed,
subsample=.8,
colsample_bytree=.9,
use_best_model=True )
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))
def compute_metrics(labels, pred_logits):
preds = pred_logits.argmax(-1)
 precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='macro')
 acc = accuracy_score(labels, preds)
 mcc = matthews_corrcoef(labels, preds)
 softmax = nn.Softmax(dim=1)
 pred_pr = softmax(torch.tensor(pred_logits))
auc = roc_auc_score(labels, pred_pr[:, 1])
 metrics = {
'mcc' : mcc,
'accuracy' : acc,
'f1' : f1,
'precision': precision,
'recall' : recall,
'auc' : auc
}
return metrics
def compute_multiclass_metrics(labels, pred_logits):

 metrics = []
n_class = labels.shape[1]
for i in range(n_class):
label = labels[:, i]
prob1 = pred_logits[:, i]
prob0 = 1 - prob1
pred = np.hstack(( prob0.reshape(( -1, 1)) , prob1.reshape(( -1, 1))))
pred = compute_metrics(label, pred)
 metrics.append(pred)
 metrics_df = pd.DataFrame(metrics)
 metrics_df = metrics_df.mean()
return metrics_df.to_dict()<train_model>
Kfolder.validate(train, test, features , lgbmodel, name="lgbfinal", prepare_stacking=True )
class MyTrainer:
def __init__(self, model, args, train_dataset, eval_dataset, compute_metrics=compute_multiclass_metrics):
self.model = model
self.args = args
self.train_dataset = train_dataset
self.eval_dataset = eval_dataset
self.compute_metrics = compute_metrics
self.isTrained = False
self.device = self.get_device_type()
self.optimizer = AdamW(model.parameters() ,
lr = args.learning_rate,
eps = args.adam_epsilon 
)
 self.epochs = self.args.num_train_epochs
self.train_dataloader, self.validation_dataloader, self.lr_scheduler, self.num_training_steps = self.get_dataLoaders()
return
def get_device_type(self):
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu")
 return device
def get_dataLoaders(self):
if isinstance(self.train_dataset, torch.utils.data.IterableDataset):
train_sampler = None
else:
train_sampler = RandomSampler(self.train_dataset)
 train_dataloader = DataLoader(
 self.train_dataset,
sampler = train_sampler,
batch_size = self.args.per_device_train_batch_size,
num_workers = 8 
)
 validation_dataloader = None
if self.eval_dataset:
validation_dataloader = DataLoader(
 self.eval_dataset,
sampler = SequentialSampler(self.eval_dataset),
batch_size = self.args.per_device_eval_batch_size,
num_workers = 8 
)
 num_training_steps = len(train_dataloader)* self.epochs
lr_scheduler = get_linear_schedule_with_warmup(self.optimizer,
num_warmup_steps = self.args.warmup_steps,
num_training_steps = num_training_steps)
 return train_dataloader, validation_dataloader, lr_scheduler, num_training_steps
def test_iterate_dataloader(self):
for step, batch in enumerate(self.train_dataloader):
print(step)
 print(batch)
 break
return
def train(self):
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 training_stats = []
total_t0 = time.time()
min_val_loss = 9999
min_train_loss = 9999
max_val_auc = -1
step = 0
for epoch_i in range(0, self.epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, self.epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
self.model.train()
for stp, batch in enumerate(self.train_dataloader):
step += 1
if step % 50 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(self.train_dataloader), elapsed))
if(self.args.max_steps > 0 and self.args.max_steps < step)or \ 
(self.args.eval_steps> 0 and step % self.args.eval_steps==0 and step>0):
avg_train_loss = total_train_loss / step
training_time = format_time(time.time() - t0)
 if self.validation_dataloader:
print("Running Validation...")
 avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate()
training_stats.append({
'epoch' : epoch_i + 1,
'training_loss' : avg_train_loss,
'eval_loss' : avg_val_loss,
'eval_f1' : avg_val_f1,
'eval_mcc' : avg_val_mcc,
'eval_precision': avg_val_precision,
'eval_recall' : avg_val_recall,
'eval_auc' : avg_val_auc,
'eval_accuracy' : avg_val_accuracy,
'training_time' : training_time,
'eval_time' : validation_time
})
 if avg_val_loss < min_val_loss:
min_val_loss = avg_val_loss
torch.save(self.model.state_dict() , "jane_emb_nn_clf_rishi.pt")
 if self.args.max_steps > 0 and self.args.max_steps < step :
print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)))
 self.isTrained = True
self.plot_train_stats_regression(training_stats)
 return training_stats
self.model.zero_grad()
', batch)
 batch['features'] = batch['features'].to(self.device ).float()
if 'labels' in batch:
batch['labels'] = batch['labels'].to(self.device ).long()
output = self.model(**batch)
 loss = output.loss
logits = output.logits
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters() , 1.0)
 self.optimizer.step()
self.lr_scheduler.step()
avg_train_loss = total_train_loss / len(self.train_dataloader)
 training_time = format_time(time.time() - t0)
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
if self.validation_dataloader:
print("
Running Validation...")
 avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate()
training_stats.append({
'epoch' : epoch_i + 1,
'training_loss' : avg_train_loss,
'eval_loss' : avg_val_loss,
'eval_f1' : avg_val_f1,
'eval_mcc' : avg_val_mcc,
'eval_precision': avg_val_precision,
'eval_recall' : avg_val_recall,
'eval_auc' : avg_val_auc,
'eval_accuracy' : avg_val_accuracy,
'training_time' : training_time,
'eval_time' : validation_time
})
 if avg_val_loss < min_val_loss:
min_val_loss = avg_val_loss
torch.save(model.state_dict() , "jane_emb_nn_clf_loss.pt")
 if avg_val_auc > max_val_auc:
max_val_auc = avg_val_auc
torch.save(self.model.state_dict() , "jane_emb_nn_clf_auc.pt")
 else:
training_stats.append({
'epoch' : epoch_i + 1,
'training_loss' : avg_train_loss,
'training_time' : training_time,
})
 if avg_train_loss < min_train_loss:
min_train_loss = avg_train_loss
torch.save(self.model.state_dict() , "jane_emb_nn_clf_rishi.pt")
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)))
 self.isTrained = True
self.model.eval()
try:
self.plot_train_stats(training_stats)
 except:
pass
torch.save(self.model.state_dict() , "jane_emb_nn_clf_end.pt")
 return training_stats
def evaluate(self):
t0 = time.time()
self.model.eval()
total_eval_mcc = 0
total_eval_f1 = 0
total_eval_precision = 0
total_eval_recall = 0
total_eval_auc = 0
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
for batch in self.validation_dataloader:
with torch.no_grad() :
batch['features'] = batch['features'].to(self.device ).float()
batch['labels'] = batch['labels'].to(self.device ).long()
output = self.model(**batch)
 loss = output.loss
logits = output.logits
total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = batch['labels'].detach().cpu().numpy()
metrics = self.compute_metrics(label_ids, logits)
 total_eval_mcc += metrics['mcc']
total_eval_f1 += metrics['f1']
total_eval_precision += metrics['precision']
total_eval_recall += metrics['recall']
total_eval_auc += metrics['auc']
total_eval_accuracy += metrics['accuracy']
avg_val_f1 = total_eval_f1 / len(self.validation_dataloader)
 print(" F1: {0:.3f}".format(avg_val_f1))
avg_val_mcc = total_eval_mcc / len(self.validation_dataloader)
 print(" MCC: {0:.3f}".format(avg_val_mcc))
avg_val_precision = total_eval_precision / len(self.validation_dataloader)
 print(" Precision: {0:.3f}".format(avg_val_precision))
avg_val_recall = total_eval_recall / len(self.validation_dataloader)
 print(" Recall: {0:.3f}".format(avg_val_recall))
avg_val_auc = total_eval_auc / len(self.validation_dataloader)
 print(" AUC: {0:.3f}".format(avg_val_auc))
avg_val_accuracy = total_eval_accuracy / len(self.validation_dataloader)
 print(" Accuracy: {0:.3f}".format(avg_val_accuracy))
avg_val_loss = total_eval_loss / len(self.validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
return avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time
def plot_train_stats(self, training_stats):

 mccs = accuracies = f1_scores = precisions = recalls = auc = losses = epochs = -1
epochs = len(training_stats)
 if 'eval_mcc' in training_stats[0]:
mccs = [e['eval_mcc'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=mccs, label='val_mcc')
 if 'eval_accuracy' in training_stats[0]:
accuracies = [e['eval_accuracy'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=accuracies, label='val_accuracy')
 if 'eval_f1' in training_stats[0]:
f1_scores = [e['eval_f1'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=f1_scores, label='val_f1')
 if 'eval_precision' in training_stats[0]:
precisions = [e['eval_precision'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=precisions, label='val_precision')
 if 'eval_recall' in training_stats[0]:
recalls = [e['eval_recall'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=recalls, label='val_recall')
 if 'eval_auc' in training_stats[0]:
auc = [e['eval_auc'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=mccs, label='val_auc')
 if 'eval_loss' in training_stats[0]:
losses = [e['eval_loss'] for e in training_stats]
if 'training_loss' in training_stats[0]:
tr_losses = [e['training_loss'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=tr_losses, label='tr_losses')
 plt.show()
print('mccs :', mccs)
 print('accuracies :', accuracies)
 print('precisions :', precisions)
 print('recalls :', recalls)
 print('f1_scores :', f1_scores)
 print('auc :', auc)
 print('losses :', losses)
 print('tr_losses :', tr_losses)
 plt.plot(recalls, precisions, marker='.', label='Prcision-Recall Curve')
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.legend()
plt.show()
return
def getTrainedModel(self):
if self.isTrained:
return self.model
return None
def predict(self, prediction_dataset, isRemoveLabels=True):

 prediction_sampler = SequentialSampler(prediction_dataset)
 prediction_dataloader = DataLoader(prediction_dataset, sampler=prediction_sampler, batch_size=self.args.per_device_eval_batch_size)
 print('Predicting labels for {:,} test sentences...'.format(len(prediction_dataset)))
 self.model.eval()
predictions = []
for batch in prediction_dataloader:
if 'labels' in batch and isRemoveLabels:
batch.pop('labels')
 batch['features'] = batch['features'].to(self.device ).float()
if 'labels' in batch:
batch['labels'] = batch['labels'].to(self.device ).long()
with torch.no_grad() :
outputs = model(**batch)
 logits = outputs.logits
logits = logits.detach().cpu().numpy()
predictions.append(logits)
 print('Done predictions for ', len(predictions), '/', len(prediction_dataloader), 'batches')
 print('Done prediction')
 pred_logits = np.concatenate(predictions, axis=0)
 return pred_logits, None, None<categorify>
xgbmodel = xgb.XGBRegressor(max_depth=5,
learning_rate=0.01,
n_estimators=10000,
objective='reg:linear',
gamma=1.45,
seed=random_seed,
silent=True,
subsample=0.8,
colsample_bytree=0.7,
colsample_bylevel=0.5 )
class JaneDataset(torch.utils.data.Dataset):
def __init__(self, df, labels=None):
if isinstance(df, pd.DataFrame):
df.reset_index(drop=True, inplace=True)
 if isinstance(labels, pd.Series):
labels.reset_index(drop=True, inplace=True)
 self.df = df
self.labels = labels
return
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
sample = None
if isinstance(self.df, pd.DataFrame):
features = np.array(self.df.loc[idx, f_columns])
 else:
features = self.df[idx]
if self.labels is not None:
sample = {'features': features, 'labels': self.labels[idx]}
else:
sample = {'features': features}
return sample<create_dataframe>
Kfolder.validate(train, test, features, xgbmodel, name="xgbfinal", prepare_stacking=True )
train_dataset = JaneDataset(X, y)
 eval_dataset = None
train_dataset[1]<normalization>
catmodel = cat.CatBoostRegressor(iterations=10000,
learning_rate=0.01,
depth=5,
eval_metric='RMSE',
colsample_bylevel=0.8,
bagging_temperature = 0.2,
metric_period = None,
early_stopping_rounds=200,
random_seed=random_seed )
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<choose_model_class>
Kfolder.validate(train, test, features , catmodel, name="catfinal", prepare_stacking=True,
fit_params={"use_best_model": True, "verbose": 100} )
class FFN(nn.Module):
def __init__(self, inputCount=160, outputCount=0, hiddenLayerCounts=[80, 40, 20], drop_prob=0.2, nonlin=nn.SiLU()):
super(FFN, self ).__init__()
self.nonlin = nonlin
self.dropout = nn.Dropout(drop_prob)
 self.dense1 = nn.Linear(inputCount, hiddenLayerCounts[0])
 self.batchnorm1 = nn.BatchNorm1d(hiddenLayerCounts[0])
 self.dense2 = nn.Linear(hiddenLayerCounts[0], hiddenLayerCounts[1])
 self.batchnorm2 = nn.BatchNorm1d(hiddenLayerCounts[1])
 self.dense3 = nn.Linear(hiddenLayerCounts[1], hiddenLayerCounts[2])
 self.batchnorm3 = nn.BatchNorm1d(hiddenLayerCounts[2])
 self.outDense = None
if outputCount > 0:
self.outDense = nn.Linear(hiddenLayerCounts[-1], outputCount)
 self.outActivtn = None
if outputCount == 1 or outputCount == 2:
self.outActivtn = nn.Sigmoid()
elif outputCount > 0:
self.outActivtn = nn.Softmax(dim=-1)
 return
def forward(self, X, **kwargs):
X = self.dropout(self.nonlin(self.batchnorm1(self.dense1(X))))
X = self.dropout(self.nonlin(self.batchnorm2(self.dense2(X))))
X = self.dropout(self.nonlin(self.batchnorm3(self.dense3(X))))
if self.outActivtn:
X = self.outActivtn(self.outDense(X))
return X<define_variables>
train['Revenue_lgb'] = train["lgbfinal"]
print("RMSE model lgb :" ,score(train, train.Revenue_lgb),)
 train['Revenue_xgb'] = train["xgbfinal"]
print("RMSE model xgb :" ,score(train, train.Revenue_xgb))
train['Revenue_cat'] = train["catfinal"]
print("RMSE model cat :" ,score(train, train.Revenue_cat))
train['Revenue_Dragon1'] = 0.4 * train["lgbfinal"] + \
0.2 * train["xgbfinal"] + \
0.4 * train["catfinal"]
print("RMSE model Dragon1 :" ,score(train, train.Revenue_Dragon1))
train['Revenue_Dragon2'] = 0.35 * train["lgbfinal"] + \
0.3 * train["xgbfinal"] + \
0.35 * train["catfinal"]
print("RMSE model Dragon2 :" ,score(train, train.Revenue_Dragon2))
class Clf_Model(nn.Module):
def __init__(self, embed_dim=N_FEAT_TAGS, csv_file='.. /input/jane-street-market-prediction/features.csv'):
super(Clf_Model, self ).__init__()
global N_FEAT_TAGS
N_FEAT_TAGS = 29
dtype = {'tag_0' : 'int8'}
for i in range(1, 29):
k = 'tag_' + str(i)
 dtype[k] = 'int8'
t_df = pd.read_csv(csv_file, usecols=range(1,N_FEAT_TAGS+1), dtype=dtype)
 t_df['tag_29'] = np.array([1] +([0] *(t_df.shape[0]-1)) ).astype('int8')
 self.features_tag_matrix = torch.tensor(t_df.to_numpy())
 N_FEAT_TAGS += 1
self.embed_dim = embed_dim
self.tag_embedding = nn.Embedding(N_FEAT_TAGS+1, embed_dim)
 self.tag_weights = nn.Linear(N_FEAT_TAGS, 1)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(inputCount=(130+embed_dim), outputCount=0, hiddenLayerCounts=[(160+embed_dim),(100+embed_dim),(50+embed_dim)], drop_prob=0.25)
 self.outDense = nn.Linear(50+embed_dim, 5)
 self.criterion = SmoothBCEwLogits(smoothing=0.005)
 return
def features2emb(self):

 all_tag_idxs = torch.LongTensor(np.arange(N_FEAT_TAGS)).to(DEVICE)
 tag_bools = self.features_tag_matrix
f_emb = self.tag_embedding(all_tag_idxs ).repeat(130, 1, 1)
 f_emb = f_emb * tag_bools[:, :, None]
s = torch.sum(tag_bools, dim=1)
 f_emb = torch.sum(f_emb, dim=-2)/ s[:, None]
return f_emb
def forward(self, features, labels=None):

 features = features.view(-1, N_FEATURES)
 if labels is not None:
labels = labels.view(-1, 5)
 f_emb = self.features2emb()
features_2 = torch.matmul(features, f_emb)
 features = torch.hstack(( features, features_2))
x = self.ffn(features)
 out_logits = self.outDense(x)
 if labels is None:
out_probs = F.sigmoid(out_logits)
 Probs = namedtuple('Probs',['logits'])
 out_probs = Probs(out_probs)
 return out_probs
batchLoss = self.criterion(out_logits, labels.float())
 out_probs = F.sigmoid(out_logits)
 Loss_Logits = namedtuple('Loss_Logits',['loss','logits'])
 loss_logits = Loss_Logits(batchLoss, out_probs)
 return loss_logits<load_pretrained>
test['revenue'] = np.expm1(test["lgbfinal"])
 test[['id','revenue']].to_csv('submission_lgb.csv', index=False)
 test[['id','revenue']].head()
model = Clf_Model()
try:
model.load_state_dict(torch.load(".. /input/jane-smooth-embnn5-160-100-50/jane_smooth_embnn5_160_100_50.pt"))
except:
model.load_state_dict(torch.load(".. /input/jane-smooth-embnn5-160-100-50/jane_smooth_embnn5_160_100_50.pt", map_location='cpu'))
model = model.float().to(DEVICE )<train_model>
test['revenue'] = np.expm1(test["xgbfinal"])
 test[['id','revenue']].to_csv('submission_xgb.csv', index=False)
 test[['id','revenue']].head()
training_args = TrainingArguments(
 output_dir = './results',
num_train_epochs= 250,
warmup_steps = 300,
eval_steps = 0,
max_steps = 0,
learning_rate = 5e-3,
per_device_train_batch_size= 80000,
per_device_eval_batch_size = 80000, 
)
 trainer = MyTrainer(
 model = model,
args = training_args,
train_dataset = train_dataset,
eval_dataset = eval_dataset,
)<train_model>
test['revenue'] = np.expm1(test["catfinal"])
 test[['id','revenue']].to_csv('submission_cat.csv', index=False)
 test[['id','revenue']].head()
<feature_engineering>
test['revenue'] = np.expm1(0.4 * test["lgbfinal"]+ 0.4 * test["catfinal"] + 0.2 * test["xgbfinal"])
 test[['id','revenue']].to_csv('submission_Dragon1.csv', index=False)
 test[['id','revenue']].head()
model.eval()
for batch in trainer.train_dataloader:
with torch.no_grad() :
batch['features'] = batch['features'].to(trainer.device ).float()
batch['labels'] = batch['labels'].to(trainer.device ).long()
output = model(**batch)
 break<predict_on_test>
test['revenue'] = np.expm1(( test["lgbfinal"] + test["catfinal"] + test["xgbfinal"])/3)
 test[['id','revenue']].to_csv('submission_Dragon2.csv', index=False)
 test[['id','revenue']].head()