Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<define_variables>
import numpy as np
import pandas as pd
import time
from datetime import datetime
top_feats = [ 1, 2, 3, 4, 5, 6, 7, 9, 11, 14, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31,
32, 33, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46,
47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 58, 59, 60,
61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
74, 75, 76, 78, 79, 80, 81, 82, 83, 84, 86, 87, 88,
89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101,
102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114,
115, 116, 117, 118, 120, 121, 122, 123, 124, 125, 126, 127, 128,
129, 130, 131, 132, 133, 136, 137, 138, 139, 140, 141, 142, 143,
144, 145, 146, 147, 149, 150, 151, 152, 153, 154, 155, 156, 157,
158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170,
171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183,
184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 197,
198, 199, 200, 202, 203, 204, 205, 206, 208, 209, 210, 211, 212,
213, 214, 215, 216, 217, 218, 219, 220, 221, 223, 224, 225, 226,
227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
240, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253,
254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266,
267, 268, 269, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280,
281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 294,
295, 296, 298, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309,
310, 311, 312, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323,
324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336,
337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349,
350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362,
363, 364, 365, 366, 367, 368, 369, 370, 371, 374, 375, 376, 377,
378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 390, 391,
392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404,
405, 406, 407, 408, 409, 411, 412, 413, 414, 415, 416, 417, 418,
419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431,
432, 434, 435, 436, 437, 438, 439, 440, 442, 443, 444, 445, 446,
447, 448, 449, 450, 453, 454, 456, 457, 458, 459, 460, 461, 462,
463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475,
476, 477, 478, 479, 481, 482, 483, 484, 485, 486, 487, 488, 489,
490, 491, 492, 493, 494, 495, 496, 498, 500, 501, 502, 503, 505,
506, 507, 509, 510, 511, 512, 513, 514, 515, 518, 519, 520, 521,
522, 523, 524, 525, 526, 527, 528, 530, 531, 532, 534, 535, 536,
538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 549, 550, 551,
552, 554, 557, 559, 560, 561, 562, 565, 566, 567, 568, 569, 570,
571, 572, 573, 574, 575, 577, 578, 580, 581, 582, 583, 584, 585,
586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 599,
600, 601, 602, 606, 607, 608, 609, 611, 612, 613, 615, 616, 617,
618, 619, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630,
631, 632, 633, 634, 635, 636, 637, 638, 639, 641, 642, 643, 644,
645, 646, 647, 648, 649, 650, 651, 652, 654, 655, 656, 658, 659,
660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672,
673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685,
686, 687, 688, 689, 691, 692, 693, 694, 695, 696, 697, 699, 700,
701, 702, 704, 705, 707, 708, 709, 710, 711, 713, 714, 716, 717,
718, 720, 721, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732,
733, 734, 735, 737, 738, 739, 740, 742, 743, 744, 745, 746, 747,
748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 759, 760, 761,
762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774,
775, 776, 777, 779, 780, 781, 782, 783, 784, 785, 786, 787, 788,
789, 790, 792, 793, 794, 795, 796, 797, 798, 800, 801, 802, 803,
804, 805, 806, 808, 809, 811, 813, 814, 815, 816, 817, 818, 819,
821, 822, 823, 825, 826, 827, 828, 829, 830, 831, 832, 834, 835,
837, 838, 839, 840, 841, 842, 845, 846, 847, 848, 850, 851, 852,
854, 855, 856, 858, 859, 860, 861, 862, 864, 866, 867, 868, 869,
870, 871, 872, 873, 874]
print(len(top_feats))<prepare_x_and_y>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
class MoaDataset(Dataset):
def __init__(self, df, targets, feats_idx, mode='train'):
self.mode = mode
self.feats = feats_idx
self.data = df[:, feats_idx]
if mode=='train':
self.targets = targets
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
if self.mode == 'train':
return torch.FloatTensor(self.data[idx]), torch.FloatTensor(self.targets[idx])
 elif self.mode == 'test':
return torch.FloatTensor(self.data[idx]), 0<define_variables>
from google.cloud import storage, automl_v1beta1 as automl
from google.api_core.gapic_v1.client_info import ClientInfo
from automlwrapper import AutoMLWrapper
train = train.values
test = test.values
train_targets = train_targets.values<prepare_x_and_y>
PROJECT_ID = 'kaggle-nlp-wdt'
BUCKET_NAME = 'kaggle-nlp-wdt-lcm'
region = 'us-central1'
storage_client = storage.Client(project=PROJECT_ID)
 client = automl.AutoMlClient(client_info=ClientInfo())
 print(f'Starting AutoML notebook at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}' )
for seed in range(nstarts):
print(f'Train seed {seed}')
 set_seed(seed)
 for n,(tr, te)in enumerate(kfold.split(train_targets, train_targets)) :
print(f'Train fold {n+1}')
 xtrain, xval = train[tr], train[te]
ytrain, yval = train_targets[tr], train_targets[te]
train_set = MoaDataset(xtrain, ytrain, top_feats)
 val_set = MoaDataset(xval, yval, top_feats)
 dataloaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True),
'val': DataLoader(val_set, batch_size=val_batch_size, shuffle=False)
 }
model = MoaModel(len(top_feats)).to(device)
 checkpoint_path = f'repeat:{seed}_Fold:{n+1}.pt'
optimizer = optim.Adam(model.parameters() , weight_decay=1e-5)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3, eps=1e-4, verbose=True)
 best_loss = {'train': np.inf, 'val': np.inf}
for epoch in range(nepochs):
epoch_loss = {'train': 0.0, 'val': 0.0}
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
for i,(x, y)in enumerate(dataloaders[phase]):
x, y = x.to(device), y.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase=='train'):
preds = model(x)
 loss = criterion(preds, y)
 if phase=='train':
loss.backward()
optimizer.step()
running_loss += loss.item() / len(dataloaders[phase])
 epoch_loss[phase] = running_loss
print("Epoch {}/{} - loss: {:5.5f} - val_loss: {:5.5f}".format(epoch+1, nepochs, epoch_loss['train'], epoch_loss['val']))
scheduler.step(epoch_loss['val'])
 if epoch_loss['val'] < best_loss['val']:
best_loss = epoch_loss
torch.save(model.state_dict() , checkpoint_path )<define_variables>
VERSION = 'V19'
BUCKET_PATH = 'preprocessing/'+VERSION+'/'
FILE_NAME = 'train_cleaned'+'_'+VERSION
training_gcs_path = BUCKET_PATH+FILE_NAME+'.csv'
dataset_display_name = FILE_NAME
model_display_name = 'model_'+FILE_NAME
oof = np.zeros(( len(train), nstarts, ntargets))
oof_targets = np.zeros(( len(train), ntargets))
preds = np.zeros(( len(test), ntargets))<compute_test_metric>
train.loc[:,['text','target']].drop_duplicates() \ 
.to_csv('train.csv', index=False, header=False )
def mean_log_loss(y_true, y_pred):
metrics = []
for i, target in enumerate(targets):
metrics.append(log_loss(y_true[:, i], y_pred[:, i].astype(float), labels=[0,1]))
return np.mean(metrics )<prepare_x_and_y>
bucket = storage.Bucket(storage_client, name=BUCKET_NAME)
 if not bucket.exists() :
bucket.create(location=BUCKET_REGION )
for seed in range(nstarts):
print(f"Inference for seed {seed}")
 seed_targets = []
seed_oof = []
seed_preds = np.zeros(( len(test), ntargets, nfolds))
for n,(tr, te)in enumerate(kfold.split(train_targets, train_targets)) :
xval, yval = train[te], train_targets[te]
fold_preds = []
val_set = MoaDataset(xval, yval, top_feats)
 test_set = MoaDataset(test, None, top_feats, mode='test')
 dataloaders = {
'val': DataLoader(val_set, batch_size=val_batch_size, shuffle=False),
'test': DataLoader(test_set, batch_size=val_batch_size, shuffle=False)
 }
checkpoint_path = f'repeat:{seed}_Fold:{n+1}.pt'
model = MoaModel(len(top_feats)).to(device)
 model.load_state_dict(torch.load(checkpoint_path))
model.eval()
for phase in ['val', 'test']:
for i,(x, y)in enumerate(dataloaders[phase]):
if phase == 'val':
x, y = x.to(device), y.to(device)
 elif phase == 'test':
x = x.to(device)
 with torch.no_grad() :
batch_preds = model(x)
 if phase == 'val':
seed_targets.append(y)
 seed_oof.append(batch_preds)
 elif phase == 'test':
fold_preds.append(batch_preds)
 fold_preds = torch.cat(fold_preds, dim=0 ).cpu().numpy()
seed_preds[:, :, n] = fold_preds
seed_targets = torch.cat(seed_targets, dim=0 ).cpu().numpy()
seed_oof = torch.cat(seed_oof, dim=0 ).cpu().numpy()
seed_preds = np.mean(seed_preds, axis=2)
 print("Score for this seed {:5.5f}".format(mean_log_loss(seed_targets, seed_oof)))
 oof_targets = seed_targets
oof[:, seed, :] = seed_oof
preds += seed_preds / nstarts
oof = np.mean(oof, axis=1)
 print("Overall score is {:5.5f}".format(mean_log_loss(oof_targets, oof)) )<save_to_csv>
def upload_blob(bucket_name, source_file_name, destination_blob_name):

 bucket = storage_client.get_bucket(bucket_name)
 blob = bucket.blob(destination_blob_name)
 blob.upload_from_filename(source_file_name)
 print('File {} uploaded to {}'.format(
 source_file_name,
'gs://' + bucket_name + '/' + destination_blob_name))
def download_to_kaggle(bucket_name,destination_directory,file_name,prefix=None):

 os.makedirs(destination_directory, exist_ok = True)
 full_file_path = os.path.join(destination_directory, file_name)
 blobs = storage_client.list_blobs(bucket_name,prefix=prefix)
 for blob in blobs:
blob.download_to_filename(full_file_path )
ss[targets] = preds
ss.loc[test_features['cp_type']=='ctl_vehicle', targets] = 0
ss.to_csv('submission.csv', index=False )<set_options>
upload_blob(BUCKET_NAME, 'train.csv', training_gcs_path )
sys.path.append('.. /input/iterativestratification')
 warnings.filterwarnings("ignore")
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<feature_engineering>
amw = AutoMLWrapper(client=client,
project_id=PROJECT_ID,
bucket_name=BUCKET_NAME,
region='us-central1',
dataset_display_name=dataset_display_name,
model_display_name=model_display_name)
def get_logger(filename='log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=f"{filename}.log")
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
logger = get_logger()
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<load_from_csv>
print(f'Getting dataset ready at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 if not amw.get_dataset_by_display_name(dataset_display_name):
print('dataset not found')
 amw.create_dataset()
amw.import_gcs_data(training_gcs_path)
 amw.dataset
print(f'Dataset ready at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train = train_features.merge(train_targets_scored, on='sig_id')
 target_cols = [c for c in train_targets_scored.columns if c not in ['sig_id']]
cols = target_cols + ['cp_type']
train[cols].groupby('cp_type' ).sum().sum(1 )<categorify>
print(f'Getting model trained at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 if not amw.get_model_by_display_name(model_display_name):
print(f'Training model at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 amw.train_model()
print(f'Model trained.Ensuring model is deployed at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 amw.deploy_model()
amw.model
print(f'Model trained and deployed at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
folds = train.copy()
Fold = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[target_cols])) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 print(folds.shape )<prepare_x_and_y>
amw.model_full_path
class TrainDataset(Dataset):
def __init__(self, df, num_features, cat_features, labels):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
self.labels = labels
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 label = torch.tensor(self.labels[idx] ).float()
return cont_x, cate_x, label
class TestDataset(Dataset):
def __init__(self, df, num_features, cat_features):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 return cont_x, cate_x
cat_features = ['cp_time', 'cp_dose']
num_features = ['g-0','g-2','g-3','g-5','g-6','g-7','g-8'
,'g-9','g-11','g-12','g-13','g-15','g-16','g-17','g-18','g-20','g-21'
,'g-22','g-24','g-25','g-26','g-27','g-28','g-29','g-30','g-31','g-32'
,'g-33','g-34','g-36','g-37','g-38','g-39','g-41','g-42','g-43','g-45'
,'g-47','g-48','g-49','g-50','g-51','g-52','g-53','g-54','g-55','g-56'
,'g-57','g-58','g-60','g-61','g-62','g-63','g-65','g-66','g-67','g-68'
,'g-69','g-70','g-71','g-72','g-73','g-75','g-76','g-77','g-78','g-79'
,'g-80','g-81','g-83','g-84','g-85','g-86','g-87','g-89','g-90','g-91'
,'g-92','g-93','g-94','g-96','g-97','g-98','g-100','g-101','g-102','g-103'
,'g-104','g-105','g-106','g-107','g-108','g-109','g-110','g-111','g-112'
,'g-113','g-114','g-115','g-116','g-117','g-118','g-119','g-120','g-121'
,'g-122','g-123','g-124','g-125','g-126','g-127','g-129','g-130','g-131'
,'g-132','g-133','g-134','g-135','g-136','g-137','g-138','g-139','g-140'
,'g-141','g-142','g-143','g-144','g-146','g-147','g-148','g-149','g-150'
,'g-151','g-152','g-154','g-156','g-157','g-158','g-160','g-161','g-162'
,'g-163','g-164','g-165','g-166','g-167','g-169','g-170','g-172','g-173'
,'g-174','g-175','g-177','g-178','g-179','g-180','g-181','g-183','g-184'
,'g-185','g-186','g-187','g-188','g-189','g-190','g-192','g-194','g-195'
,'g-196','g-199','g-200','g-202','g-203','g-205','g-206','g-207','g-208'
,'g-209','g-210','g-211','g-212','g-215','g-216','g-217','g-218','g-219'
,'g-221','g-222','g-224','g-225','g-226','g-227','g-228','g-229','g-230'
,'g-231','g-233','g-235','g-236','g-237','g-238','g-239','g-240','g-241'
,'g-242','g-243','g-245','g-246','g-247','g-248','g-250','g-251','g-252'
,'g-253','g-254','g-255','g-256','g-257','g-258','g-260','g-262','g-263'
,'g-265','g-267','g-268','g-269','g-270','g-272','g-273','g-274','g-276'
,'g-279','g-280','g-283','g-284','g-285','g-286','g-287','g-291','g-292'
,'g-293','g-294','g-296','g-297','g-298','g-299','g-300','g-301','g-302'
,'g-303','g-305','g-306','g-307','g-308','g-309','g-310','g-312','g-313'
,'g-314','g-317','g-318','g-319','g-321','g-322','g-323','g-324','g-325'
,'g-326','g-327','g-328','g-329','g-330','g-331','g-332','g-335','g-336'
,'g-337','g-338','g-340','g-341','g-342','g-343','g-344','g-346','g-347'
,'g-348','g-349','g-350','g-352','g-353','g-354','g-355','g-356','g-357'
,'g-358','g-359','g-360','g-361','g-362','g-363','g-365','g-366','g-367'
,'g-368','g-369','g-371','g-372','g-373','g-374','g-375','g-376','g-377'
,'g-379','g-380','g-381','g-382','g-383','g-384','g-385','g-386','g-387'
,'g-388','g-389','g-390','g-391','g-392','g-394','g-395','g-396','g-397'
,'g-398','g-400','g-402','g-403','g-404','g-405','g-407','g-408','g-409'
,'g-410','g-411','g-412','g-414','g-415','g-416','g-417','g-418','g-419'
,'g-420','g-421','g-422','g-423','g-424','g-425','g-426','g-427','g-428'
,'g-429','g-430','g-431','g-432','g-433','g-434','g-435','g-438','g-439'
,'g-440','g-441','g-442','g-443','g-444','g-445','g-446','g-447','g-449'
,'g-450','g-451','g-453','g-454','g-455','g-456','g-457','g-458','g-459'
,'g-460','g-461','g-462','g-463','g-465','g-466','g-468','g-469','g-470'
,'g-471','g-472','g-473','g-474','g-475','g-476','g-479','g-480','g-482'
,'g-483','g-484','g-485','g-486','g-488','g-489','g-491','g-492','g-493'
,'g-497','g-498','g-499','g-500','g-502','g-503','g-504','g-506','g-507'
,'g-508','g-509','g-510','g-511','g-513','g-514','g-515','g-516','g-518'
,'g-520','g-522','g-523','g-524','g-525','g-526','g-527','g-528','g-529'
,'g-530','g-531','g-533','g-534','g-535','g-536','g-537','g-538','g-539'
,'g-540','g-541','g-542','g-543','g-544','g-546','g-547','g-550','g-551'
,'g-552','g-553','g-554','g-555','g-556','g-557','g-558','g-559','g-560'
,'g-561','g-562','g-563','g-564','g-566','g-567','g-568','g-569','g-570'
,'g-571','g-572','g-574','g-577','g-578','g-579','g-580','g-583','g-584'
,'g-587','g-588','g-589','g-590','g-592','g-593','g-594','g-595','g-596'
,'g-597','g-598','g-599','g-600','g-602','g-604','g-605','g-606','g-608'
,'g-609','g-610','g-611','g-612','g-613','g-614','g-616','g-619','g-620'
,'g-622','g-624','g-627','g-628','g-629','g-630','g-631','g-632','g-634'
,'g-635','g-636','g-639','g-640','g-641','g-642','g-643','g-644','g-646'
,'g-647','g-648','g-649','g-651','g-652','g-655','g-656','g-657','g-658'
,'g-659','g-660','g-661','g-663','g-664','g-665','g-666','g-667','g-669'
,'g-671','g-672','g-673','g-674','g-675','g-677','g-678','g-679','g-681'
,'g-682','g-683','g-684','g-685','g-686','g-688','g-689','g-691','g-692'
,'g-693','g-694','g-696','g-697','g-698','g-699','g-700','g-701','g-702'
,'g-704','g-705','g-706','g-708','g-709','g-710','g-711','g-712','g-713'
,'g-714','g-720','g-722','g-724','g-725','g-726','g-727','g-728','g-729'
,'g-731','g-733','g-734','g-735','g-736','g-737','g-738','g-739','g-740'
,'g-741','g-742','g-743','g-744','g-745','g-746','g-747','g-748','g-749'
,'g-750','g-751','g-752','g-753','g-755','g-756','g-757','g-758','g-759'
,'g-760','g-761','g-762','g-763','g-764','g-766','g-767','g-768','g-769'
,'g-771','c-0','c-5','c-6','c-7','c-8','c-9','c-10','c-12','c-13','c-15'
,'c-18','c-20','c-22','c-24','c-25','c-26','c-30','c-33','c-34','c-36'
,'c-37','c-38','c-41','c-44','c-45','c-46','c-47','c-48','c-50','c-51'
,'c-52','c-54','c-56','c-57','c-58','c-59','c-60','c-62','c-63','c-64'
,'c-65','c-66','c-67','c-69','c-70','c-71','c-72','c-73','c-75','c-76'
,'c-77','c-79','c-80','c-81','c-83','c-85','c-86','c-87','c-89','c-92'
,'c-93','c-95','c-96','c-98','c-99']
target = train[target_cols].values
def cate2num(df):
df['cp_time'] = df['cp_time'].map({24: 0, 48: 1, 72: 2})
 df['cp_dose'] = df['cp_dose'].map({'D1': 3, 'D2': 4})
 return df
train = cate2num(train)
 test = cate2num(test )<choose_model_class>
print(f'Begin getting predictions at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 prediction_client = automl.PredictionServiceClient()
amw.set_prediction_client(prediction_client)
 predictions_df = amw.get_predictions(test,
input_col_name='text',
limit=None,
threshold=0.5,
verbose=False)
 print(f'Finished getting predictions at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}' )
class CFG:
max_grad_norm=1000
gradient_accumulation_steps=1
hidden_size=512
dropout=0.45
lr=1e-3
weight_decay=1e-5
batch_size=128
epochs=50
num_features=num_features
cat_features=cat_features
target_cols=target_cols
class TabularNN(nn.Module):
def __init__(self, cfg):
super().__init__()
self.mlp = nn.Sequential(
 nn.Linear(len(cfg.num_features), cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, len(cfg.target_cols)) , 
)
 def forward(self, cont_x, cate_x):
x = self.mlp(cont_x)
 return x
def train_fn(train_loader, model, optimizer, epoch, scheduler, device):
losses = AverageMeter()
model.train()
for step,(cont_x, cate_x, y)in enumerate(train_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
loss.backward()
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters() , CFG.max_grad_norm)
 if(step + 1)% CFG.gradient_accumulation_steps == 0:
scheduler.step()
optimizer.step()
optimizer.zero_grad()
return losses.avg
def validate_fn(valid_loader, model, device):
losses = AverageMeter()
model.eval()
val_preds = []
for step,(cont_x, cate_x, y)in enumerate(valid_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 val_preds.append(pred.sigmoid().detach().cpu().numpy())
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
val_preds = np.concatenate(val_preds)
 return losses.avg, val_preds
def inference_fn(test_loader, model, device):
model.eval()
preds = []
for step,(cont_x, cate_x)in enumerate(test_loader):
cont_x, cate_x = cont_x.to(device), cate_x.to(device)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 preds.append(pred.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def run_single_nn(cfg, train, test, folds, num_features, cat_features, target, device, fold_num=0, seed=42):
logger.info(f'Set seed {seed}')
 seed_everything(seed=seed)
 trn_idx = folds[folds['fold'] != fold_num].index
val_idx = folds[folds['fold'] == fold_num].index
train_folds = train.loc[trn_idx].reset_index(drop=True)
 valid_folds = train.loc[val_idx].reset_index(drop=True)
 train_target = target[trn_idx]
valid_target = target[val_idx]
train_dataset = TrainDataset(train_folds, num_features, cat_features, train_target)
 valid_dataset = TrainDataset(valid_folds, num_features, cat_features, valid_target)
 train_loader = DataLoader(train_dataset, batch_size=cfg.batch_size, shuffle=True,
num_workers=4, pin_memory=True, drop_last=True)
 valid_loader = DataLoader(valid_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True, drop_last=False)
 model = TabularNN(cfg)
 model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=cfg.lr)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=cfg.epochs, steps_per_epoch=len(train_loader))
log_df = pd.DataFrame(columns=(['EPOCH']+['TRAIN_LOSS']+['VALID_LOSS']))
best_loss = np.inf
for epoch in range(cfg.epochs):
train_loss = train_fn(train_loader, model, optimizer, epoch, scheduler, device)
 valid_loss, val_preds = validate_fn(valid_loader, model, device)
 log_row = {'EPOCH': epoch,
'TRAIN_LOSS': train_loss,
'VALID_LOSS': valid_loss,
}
log_df = log_df.append(pd.DataFrame(log_row, index=[0]), sort=False)
 if valid_loss < best_loss:
logger.info(f'epoch{epoch} save best model...{valid_loss}')
 best_loss = valid_loss
oof = np.zeros(( len(train), len(cfg.target_cols)))
 oof[val_idx] = val_preds
torch.save(model.state_dict() , f"fold{fold_num}_seed{seed}.pth")
 test_dataset = TestDataset(test, num_features, cat_features)
 test_loader = DataLoader(test_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True)
 model = TabularNN(cfg)
 model.load_state_dict(torch.load(f"fold{fold_num}_seed{seed}.pth"))
model.to(device)
 predictions = inference_fn(test_loader, model, device)
 torch.cuda.empty_cache()
return oof, predictions
def run_kfold_nn(cfg, train, test, folds, num_features, cat_features, target, device, n_fold=5, seed=42):
oof = np.zeros(( len(train), len(cfg.target_cols)))
 predictions = np.zeros(( len(test), len(cfg.target_cols)))
 for _fold in range(n_fold):
logger.info("Fold {}".format(_fold))
_oof, _predictions = run_single_nn(cfg,
train,
test,
folds,
num_features,
cat_features,
target,
device,
fold_num=_fold,
seed=seed)
 oof += _oof
predictions += _predictions / n_fold
score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"CV score: {score}")
 return oof, predictions
def just_predict(cfg, test_loader, model_paths, device):
test_dataset = TestDataset(test, num_features, cat_features)
 test_loader = DataLoader(test_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True)
 predictions = np.zeros(( len(test), len(cfg.target_cols)))
 for path in model_paths:
model = TabularNN(cfg)
 model.load_state_dict(torch.load(path))
model.to(device)
 prediction = inference_fn(test_loader, model, device)
 predictions += prediction / len(model_paths)
 print('Done! ', path)
 return predictions<predict_on_test>
submission_df = pd.concat([test['id'], predictions_df['class']], axis=1 )
model_paths = glob.glob('.. /input/moa-model-weights/pth_0908_dr45/*.pth')
 predictions = just_predict(CFG, test, model_paths, device)
 test[target_cols] = predictions
test<save_to_csv>
submission_df = submission_df.rename(columns={'class':'target'})
 submission_df.head()
<save_to_csv><EOS> 
submission_df.to_csv("submission.csv", index=False, header=True )