Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<drop_column>
warnings.filterwarnings('ignore' )
del X_train, X_valid
del target, valid_prices
del epochs, lr_init, lr_fin, exp_decay, lr_decay
gc.collect()<load_from_csv>
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
t1 = time.time()
train = pd.read_csv('.. /input/mercari-price-suggestion-challenge/train.tsv', sep='\t')
 train = train[train['price'] != 0]
train['target'] = np.log1p(train['price'])
 n_train = train.shape[0]<data_type_conversions>
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape))
def split_cat(text):
try:
s1, s2, s3 = text.split("/")
 return s1, s2, s3
except: return("No Label", "No Label", "No Label")
 def wordbatch_preprocess(df):
df["category_name"] = df["category_name"].fillna(value="missing" ).astype(str)
 df["name"] = df["name"].fillna(value="missing" ).astype(str)
 df["brand_name"] = df["brand_name"].fillna(value="missing" ).astype(str)
 df["item_description"] = df["item_description"].fillna(value="missing" ).astype(str)
 df["item_condition_id"] = df["item_condition_id"].astype(int)
 df["shipping"] = df["shipping"].astype(int)
 print("Got so far")
 print(df.dtypes)
 df['subcat_0'], df['subcat_1'], df['subcat_2'] = zip(*df['category_name'].apply(lambda x: split_cat(x)))
 print(df.dtypes)
 return df
train = wordbatch_preprocess(train)
 print("Pre Process for Word batch is done" )<train_model>
train_features2=train_features.copy()
test_features2=test_features.copy()
class WordBatchModel(object):
def __init__(self):
self.wb_desc = None
self.desc_indices = None
self.cv_name, self.cv_name2 = None, None
self.cv_cat0, self.cv_cat1, self.cv_cat2 = None, None, None
self.cv_brand = None
self.cv_condition = None
self.cv_cat_brand = None
self.desc3 = None
self.model = None
def train_wbm(self, df):
self.wb_desc = wordbatch.WordBatch(None,
extractor=(WordBag, {"hash_ngrams": 2, "hash_ngrams_weights": [1.0, 1.0],
"hash_size": 2 ** 28, "norm": "l2", "tf": 1.0,
"idf": None}), procs=8)
 self.wb_desc.dictionary_freeze = True
X_desc = self.wb_desc.fit_transform(df['item_description'])
 self.desc_indices = np.array(np.clip(X_desc.getnnz(axis=0)- 1, 0, 1), dtype=bool)
 X_desc = X_desc[:, self.desc_indices]
self.cv_cat0 = CountVectorizer(min_df=2)
 X_category0 = self.cv_cat0.fit_transform(df['subcat_0'])
 self.cv_cat1 = CountVectorizer(min_df=2)
 X_category1 = self.cv_cat1.fit_transform(df['subcat_1'])
 self.cv_cat2 = CountVectorizer(min_df=2)
 X_category2 = self.cv_cat2.fit_transform(df['subcat_2'])
 self.cv_brand = CountVectorizer(min_df=2, token_pattern=".+")
 X_brand = self.cv_brand.fit_transform(df['brand_name'])
 self.cv_name = CountVectorizer(min_df=2, ngram_range=(1, 1),binary=True, token_pattern="\w+")
 X_name = 2 * self.cv_name.fit_transform(df['name'])
 self.cv_name2 = CountVectorizer(min_df=2, ngram_range=(2, 2), binary=True, token_pattern="\w+")
 X_name2 = 0.5 * self.cv_name2.fit_transform(df['name'])
 df["cat_brand"] = [a + " " + b for a, b in zip(df["category_name"], df["brand_name"])]
self.cv_cat_brand = CountVectorizer(min_df=10, token_pattern=".+")
 X_cat_brand = self.cv_cat_brand.fit_transform(df["cat_brand"])
 self.cv_condition = CountVectorizer(token_pattern=".+")
 X_condition = self.cv_condition.fit_transform(( df['item_condition_id'] + 10 * df["shipping"] ).apply(str))
self.desc3 = CountVectorizer(ngram_range=(3, 3), max_features=1000, binary=True, token_pattern="\w+")
 X_desc3 = self.desc3.fit_transform(df["item_description"])
 X = hstack(( X_condition,
X_desc, X_brand,
X_category0, X_category1, X_category2,
X_name, X_name2,
X_cat_brand,
X_desc3 
)).tocsr()
print("X Reconstructed")
 y = df['target'].values
print("Y created")
 self.model = FM_FTRL(alpha=0.01, beta=0.01, L1=0.00001, L2=0.1, D=X.shape[1], alpha_fm=0.02, L2_fm=0.0,
init_fm=0.01, D_fm=200, e_noise=0.0001, iters=10, inv_link="identity", threads=4)
 print("Model Defined")
 self.model.fit(X, y)
 def predict(self, df):
X_desc = self.wb_desc.transform(df["item_description"])
 X_desc = X_desc[:, self.desc_indices]
X_brand = self.cv_brand.transform(df['brand_name'])
 X_name = 2 * self.cv_name.transform(df["name"])
 X_name2 = 0.5 * self.cv_name2.transform(df["name"])
 X_category0 = self.cv_cat0.transform(df['subcat_0'])
 X_category1 = self.cv_cat1.transform(df['subcat_1'])
 X_category2 = self.cv_cat2.transform(df['subcat_2'])
 X_condition = self.cv_condition.transform(( df['item_condition_id'] + 10 * df["shipping"] ).apply(str))
df["cat_brand"] = [a + " " + b for a, b in zip(df["category_name"], df["brand_name"])]
X_cat_brand = self.cv_cat_brand.transform(df["cat_brand"])
 X_desc3 = self.desc3.transform(df["item_description"])
 X = hstack(( X_condition,
X_desc, X_brand,
X_category0, X_category1, X_category2,
X_name, X_name2,
X_cat_brand,
X_desc3 
)).tocsr()
return self.model.predict(X )<train_model>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
os.environ['MKL_NUM_THREADS'] = '4'
os.environ['OMP_NUM_THREADS'] = '4'
print(train.shape)
 t1 = time.time()
wbm = WordBatchModel()
wbm.train_wbm(train)
 t2 = time.time()
print("Time taken for Word Batch is "+str(t2-t1))
del train
gc.collect()<load_from_csv>
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 data = pd.concat([pd.DataFrame(train_features[GENES+CELLS]), pd.DataFrame(test_features[GENES+CELLS])])
 data2 = qt.fit_transform(data[GENES+CELLS])
 train_features[GENES+CELLS] = pd.DataFrame(data2[:train_features.shape[0]])
 test_features[GENES+CELLS] = pd.DataFrame(data2[-test_features.shape[0]:] )
t1 = time.time()
def load_test() :
for df in pd.read_csv('.. /input/mercari-price-suggestion-challenge/test_stg2.tsv', sep='\t', chunksize= 1000000):
yield df
test_ids = np.array([], dtype=np.int32)
 preds= np.array([], dtype=np.float32)
 i = 0
for df in load_test() :
i +=1
print(" Chunk number is "+str(i))
df1 = df2 = df
testRNN = preprocess_RNN(df1)
 print(df1.dtypes)
 print(testRNN.dtypes)
 testRNN = transform_RNN(testRNN, le_name, le_brand, word_token)
 print(testRNN.dtypes)
 X_testRNN = get_keras_data(testRNN)
 predsRNN1 = modelRNN.predict(X_testRNN, batch_size = BATCH_SIZE, verbose = 1)
 test_id = df['test_id']
del testRNN
del df['seq_item_description'], df['seq_name'], df['brand'], df['category']
gc.collect()
print("RNN Prediction is done")
 print(df.isnull().any())
 print(df.dtypes)
 test_WB = wordbatch_preprocess(df2)
 print(df.dtypes)
 print("Word batch preprocess done")
 predsWB1 = wbm.predict(test_WB)
 print("Word Batch Prediction done")
 predsRNN = np.expm1(predsRNN1)
 predsWB = np.expm1(predsWB1)
 predsRNN = predsRNN.reshape(predsRNN.shape[0],1)
 predsWB = predsWB.reshape(predsWB.shape[0],1)
 predsRNN = np.clip(predsRNN, 0, predsRNN.max())
 predsWB = np.clip(predsWB, 0, predsWB.max())
 preds= np.append(preds,(( predsRNN*0.4)+(predsWB*0.6)))
 test_ids = np.append(test_ids, test_id)
 print("All chunks done")
 t2 = time.time()
print("Total time for Parallel Batch Prediction is "+str(t2-t1))<save_to_csv>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )
submission = pd.DataFrame(columns = ['test_id', 'price'])
 submission['test_id'] = test_ids
submission['price'] = preds
print("Check Submission NOW!!!!!!!!@")
 submission.to_csv("BatchPrediction_MemoryCoreOptimized.csv", index=False )<train_model>
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1 )
t2 = time.time()
print("Total time taken is "+str(t2-start_time))<install_modules>
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1 )
!pip install -q efficientnet_pytorch > /dev/null 
<set_options>
var_thresh = VarianceThreshold(0.85)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<choose_model_class>
def fe_cluster(train, test, n_clusters_g = 22, n_clusters_c = 4, SEED = 42):
features_g = GENES
features_c = CELLS
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster(train_features2,test_features2 )
def get_net() :
net = EfficientNet.from_pretrained('efficientnet-b2')
 net._fc = nn.Linear(in_features=1408, out_features=4, bias=True)
 return net
net = get_net().cuda()<load_pretrained>
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )
checkpoint = torch.load('.. /input/eb2-weights/best-checkpoint-038epoch_c.bin')
 net.load_state_dict(checkpoint['model_state_dict']);
net.eval() ;<data_type_conversions>
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_pca'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_pca'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )
class DatasetSubmissionRetriever(Dataset):
def __init__(self, image_names, transforms=None):
super().__init__()
self.image_names = image_names
self.transforms = transforms
def __getitem__(self, index: int):
image_name = self.image_names[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/Test/{image_name}', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image_name, image
def __len__(self)-> int:
return self.image_names.shape[0]<define_variables>
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]
DATA_ROOT_PATH = '.. /input/alaska2-image-steganalysis'
results = []
for mode in range(0, 4):
dataset = DatasetSubmissionRetriever(
 image_names=np.array([path.split('/')[-1] for path in glob('.. /input/alaska2-image-steganalysis/Test/*.jpg')]),
transforms=get_test_transforms(mode), 
)
 data_loader = DataLoader(
 dataset,
batch_size=8,
shuffle=False,
num_workers=2,
drop_last=False, 
)
 result = {'Id': [], 'Label': []}
for step,(image_names, images)in enumerate(data_loader):
print(step, end='\r')
 y_pred = net(images.cuda())
 y_pred = 1 - nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,0]
result['Id'].extend(image_names)
 result['Label'].extend(y_pred)
 results.append(result )<save_to_csv>
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]
y_pred = net(images.cuda())
 y_pred = 1 - nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,0]
result['Id'].extend(image_names)
 result['Label'].extend(y_pred)
 submissions = []
for mode in range(0,4):
submission = pd.DataFrame(results[mode])
 submissions.append(submission)
 for mode in range(0,4):
submissions[mode].to_csv(f'submission_{mode}.csv', index=False)
 submissions[0]['Label'] =(submissions[0]['Label']*3 + submissions[1]['Label']*1 + submissions[2]['Label']*1 + submissions[3]['Label']*1)/ 6
submissions[0].to_csv(f'submission_B2_c.csv', index=False )<install_modules>
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
!pip install -q catalyst > /dev/null
!pip install -q lycon > /dev/null
!git clone https://github.com/NVIDIA/apex
!pip install -v --no-cache-dir --global-option="--cuda_ext"./apex/
!free -g
!nvidia-smi<import_modules>
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )
SEED = 42
EPS = 1e-8
REBUILD_16X16_DCT_SUMS = False
REBUILD_16X16_PXL_SUMS = False
REBUILD_JPEG_CACHE = False
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<define_variables>
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]
DATA_ROOT_PATH = '.. /input/alaska2-image-steganalysis' 
<categorify>
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )
def get_valid_transforms() :
return A.Compose([
A.Normalize(always_apply=True),
ToTensorV2(p=1.0),
], p=1.0 )<import_modules>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True )
GlobalParams = collections.namedtuple('GlobalParams', [
'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',
'num_classes', 'width_coefficient', 'depth_coefficient',
'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])
 BlockArgs = collections.namedtuple('BlockArgs', [
'kernel_size', 'num_repeat', 'input_filters', 'output_filters',
'expand_ratio', 'id_skip', 'stride', 'se_ratio'])
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields)
 class SwishImplementation(torch.autograd.Function):
@staticmethod
def forward(ctx, i):
result = i * torch.sigmoid(i)
 ctx.save_for_backward(i)
 return result
@staticmethod
def backward(ctx, grad_output):
i = ctx.saved_variables[0]
sigmoid_i = torch.sigmoid(i)
 return grad_output *(sigmoid_i *(1 + i *(1 - sigmoid_i)))
 class MemoryEfficientSwish(nn.Module):
def forward(self, x):
return SwishImplementation.apply(x)
 class Swish(nn.Module):
def forward(self, x):
return x * torch.sigmoid(x)
 def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
if not multiplier:
return filters
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def drop_connect(inputs, p, training):

 if not training: return inputs
batch_size = inputs.shape[0]
keep_prob = 1 - p
random_tensor = keep_prob
random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)
 binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
def get_same_padding_conv2d(image_size=None):

 if image_size is None:
return Conv2dDynamicSamePadding
else:
return partial(Conv2dStaticSamePadding, image_size=image_size)
 class Conv2dDynamicSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):
super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])
 return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 class Conv2dStaticSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):
super().__init__(in_channels, out_channels, kernel_size, **kwargs)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
assert image_size is not None
ih, iw = image_size if type(image_size)== list else [image_size, image_size]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(( pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 return x
class Identity(nn.Module):
def __init__(self,):
super(Identity, self ).__init__()
def forward(self, input):
return input
def efficientnet_params(model_name):

 params_dict = {
'efficientnet-b0':(1.0, 1.0, 224, 0.2),
'efficientnet-b1':(1.0, 1.1, 240, 0.2),
'efficientnet-b2':(1.1, 1.2, 260, 0.3),
'efficientnet-b3':(1.2, 1.4, 300, 0.3),
'efficientnet-b4':(1.4, 1.8, 380, 0.4),
'efficientnet-b5':(1.6, 2.2, 456, 0.4),
'efficientnet-b6':(1.8, 2.6, 528, 0.5),
'efficientnet-b7':(2.0, 3.1, 600, 0.5),
'efficientnet-b8':(2.2, 3.6, 672, 0.5),
'efficientnet-l2':(4.3, 5.3, 800, 0.5),
}
return params_dict[model_name]
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split('_')
 options = {}
for op in ops:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
assert(( 's' in options and len(options['s'])== 1)or 
(len(options['s'])== 2 and options['s'][0] == options['s'][1]))
return BlockArgs(
 kernel_size=int(options['k']),
num_repeat=int(options['r']),
input_filters=int(options['i']),
output_filters=int(options['o']),
expand_ratio=int(options['e']),
id_skip=('noskip' not in block_string),
se_ratio=float(options['se'])if 'se' in options else None,
stride=[int(options['s'][0])])
 @staticmethod
def _encode_block_string(block):

 args = [
'r%d' % block.num_repeat,
'k%d' % block.kernel_size,
's%d%d' %(block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
 if block.id_skip is False:
args.append('noskip')
 return '_'.join(args)
 @staticmethod
def decode(string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(BlockDecoder._decode_block_string(block_string))
return blocks_args
@staticmethod
def encode(blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(BlockDecoder._encode_block_string(block))
return block_strings
def efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,
drop_connect_rate=0.2, image_size=None, num_classes=1000):

 blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',
'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',
'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',
'r1_k3_s11_e6_i192_o320_se0.25',
]
blocks_args = BlockDecoder.decode(blocks_args)
 global_params = GlobalParams(
 batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
dropout_rate=dropout_rate,
drop_connect_rate=drop_connect_rate,
num_classes=num_classes,
width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
depth_divisor=8,
min_depth=None,
image_size=image_size, 
)
 return blocks_args, global_params
def get_model_params(model_name, override_params):

 if model_name.startswith('efficientnet'):
w, d, s, p = efficientnet_params(model_name)
 blocks_args, global_params = efficientnet(
 width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)
 else:
raise NotImplementedError('model name is not pre-defined: %s' % model_name)
 if override_params:
global_params = global_params._replace(**override_params)
 return blocks_args, global_params
url_map = {
'efficientnet-b0': '.. /input/noisy-student-efficientnet-b0.pth',
'efficientnet-b1': '.. /input/noisy-student-efficientnet-b1.pth',
'efficientnet-b2': '.. /input/noisy-student-efficientnet-b2.pth',
'efficientnet-b3': '.. /input/noisy-student-efficientnet-b3.pth',
'efficientnet-b4': '.. /input/noisy-student-efficientnet-b4.pth',
'efficientnet-b5': '.. /input/noisy-student-efficientnet-b5.pth',
'efficientnet-b6': '.. /input/noisy-student-efficientnet-b6.pth',
'efficientnet-b7': '.. /input/noisy-student-efficientnet-b7.pth',
}
url_map_advprop = {
'efficientnet-b0': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b0-b64d5a18.pth',
'efficientnet-b1': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b1-0f3ce85a.pth',
'efficientnet-b2': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b2-6e9d97e5.pth',
'efficientnet-b3': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b3-cdd7c0f4.pth',
'efficientnet-b4': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b4-44fb3a87.pth',
'efficientnet-b5': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b5-86493f6b.pth',
'efficientnet-b6': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b6-ac80338e.pth',
'efficientnet-b7': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b7-4652b6dd.pth',
'efficientnet-b8': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b8-22a8fe65.pth',
}
def load_pretrained_weights(model, model_name, load_fc=True, advprop=False):

 url_map_ = url_map_advprop if advprop else url_map
state_dict = torch.load(url_map_[model_name])
 if load_fc:
model.load_state_dict(state_dict)
 else:
state_dict.pop('_fc.weight')
 state_dict.pop('_fc.bias')
 res = model.load_state_dict(state_dict, strict=False)
 print(res)
 print('Loaded pretrained weights for {}'.format(model_name)) 
<statistical_test>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
def DCT_Basis() :
N = 8
basis = []
for u in range(8):
for v in range(8):
z = np.zeros(( N,N))
for i in range(N):
for j in range(N):
z[i,j] = np.cos(np.pi*(2*i+1)*u /(2*N)) * np.cos(np.pi*(2*j+1)*v /(2*N))
basis.append(z)
 return torch.Tensor([basis,basis,basis] ).permute(( 1,0,2,3))
def gem(x, p=3, eps=1e-6):
x = x.double()
x = F.avg_pool2d(x.clamp(min=eps ).pow(p),(x.size(-2), x.size(-1)) ).pow(1./p)
 return x.half()
class GeM(nn.Module):
def __init__(self, p=3, eps=1e-6):
super(GeM,self ).__init__()
self.p = nn.Parameter(torch.ones(1)*p)
 self.eps = eps
def forward(self, x):
return gem(x, self.p, self.eps)
 def __repr__(self):
return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist() [0])+ ', ' + 'eps=' + str(self.eps)+ ')'
class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params):
super().__init__()
self._block_args = block_args
self._bn_mom = 1 - global_params.batch_norm_momentum
self._bn_eps = global_params.batch_norm_epsilon
self.has_se =(self._block_args.se_ratio is not None)and(0 < self._block_args.se_ratio <= 1)
 self.id_skip = block_args.id_skip
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 inp = self._block_args.input_filters
oup = self._block_args.input_filters * self._block_args.expand_ratio
if self._block_args.expand_ratio != 1:
self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 k = self._block_args.kernel_size
s = self._block_args.stride
self._depthwise_conv = Conv2d(
 in_channels=oup, out_channels=oup, groups=oup,
kernel_size=k, stride=s, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 if self.has_se:
num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))
self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)
 self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)
 final_oup = self._block_args.output_filters
self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)
 self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)
 self._swish = MemoryEfficientSwish()
def set_trainable(self, trainable):
self._expand_conv.requires_grad_(trainable)
 self._bn0.requires_grad_(trainable)
 self._depthwise_conv.requires_grad_(trainable)
 self._bn1.requires_grad_(trainable)
 self._se_reduce.requires_grad_(trainable)
 self._se_expand.requires_grad_(trainable)
 self._project_conv.requires_grad_(trainable)
 self._bn2.requires_grad_(trainable)
 def forward(self, inputs, drop_connect_rate=None):

 x = inputs
if self._block_args.expand_ratio != 1:
x = self._swish(self._bn0(self._expand_conv(inputs)))
 x = self._swish(self._bn1(self._depthwise_conv(x)))
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_expand(self._swish(self._se_reduce(x_squeezed)))
 x = torch.sigmoid(x_squeezed)* x
x = self._bn2(self._project_conv(x))
input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters
if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x, p=drop_connect_rate, training=self.training)
 x = x + inputs
return x
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
class EfficientNet(nn.Module):

 def __init__(self, blocks_args=None, global_params=None):
super().__init__()
assert isinstance(blocks_args, list), 'blocks_args should be a list'
assert len(blocks_args)> 0, 'block args must be greater than 0'
self._global_params = global_params
self._blocks_args = blocks_args
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 bn_mom = 1 - self._global_params.batch_norm_momentum
bn_eps = self._global_params.batch_norm_epsilon
in_channels = 3
out_channels = round_filters(32, self._global_params)
 self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._blocks = nn.ModuleList([])
 for block_args in self._blocks_args:
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, self._global_params),
output_filters=round_filters(block_args.output_filters, self._global_params),
num_repeat=round_repeats(block_args.num_repeat, self._global_params)
)
 self._blocks.append(MBConvBlock(block_args, self._global_params))
if block_args.num_repeat > 1:
block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self._global_params))
in_channels = block_args.output_filters
out_channels = round_filters(1280, self._global_params)
 self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._pxls = nn.Conv2d(out_channels, 1, 1)
 self._dropout = nn.Dropout(self._global_params.dropout_rate)
 self._fc = nn.Linear(out_channels, self._global_params.num_classes)
 self._fc_aux = nn.Linear(out_channels, 2)
 self._swish = MemoryEfficientSwish()
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
for block in self._blocks:
block.set_swish(memory_efficient)
 def extract_features(self, inputs):

 x = self._swish(self._bn0(self._conv_stem(inputs)))
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 x = self._swish(self._bn1(self._conv_head(x)))
 return x
def forward(self, inputs):

 bs = inputs.size(0)
 x = self.extract_features(inputs)
 pxls = self._pxls(x)
 x = F.adaptive_avg_pool2d(x, 1)
 x = x.view(bs, -1)
 x = self._dropout(x)
 aux = self._fc_aux(x)
 x = self._fc(x)
 return x, aux, pxls
@classmethod
def from_name(cls, model_name, override_params=None):
cls._check_model_name_is_valid(model_name)
 blocks_args, global_params = get_model_params(model_name, override_params)
 return cls(blocks_args, global_params)
 @classmethod
def from_pretrained(cls, model_name, advprop=False, num_classes=1000, in_channels=3):
model = cls.from_name(model_name, override_params={'num_classes': num_classes})
 load_pretrained_weights(model, model_name, load_fc=(num_classes == 1000), advprop=advprop)
 if in_channels != 3:
Conv2d = get_same_padding_conv2d(image_size = model._global_params.image_size)
 out_channels = round_filters(32, model._global_params)
 model._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 return model
@classmethod
def get_image_size(cls, model_name):
cls._check_model_name_is_valid(model_name)
 _, _, res, _ = efficientnet_params(model_name)
 return res
@classmethod
def _check_model_name_is_valid(cls, model_name):

 valid_models = ['efficientnet-b'+str(i)for i in range(9)]
if model_name not in valid_models:
raise ValueError('model_name should be one of: ' + ', '.join(valid_models)) 
<load_pretrained>
target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))
checkpoint = torch.load('.. /input/alaska25/best-checkpoint-040epoch.bin')['model_state_dict']
for key in list(checkpoint.keys()):
nkey = key[len('module.'):]
checkpoint[nkey] = checkpoint[key]
del checkpoint[key]<find_best_params>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
net.load_state_dict(checkpoint)
 net.eval()
net = net.cuda()
net = amp.initialize(net, opt_level='O1')
 net = torch.nn.DataParallel(net, device_ids=[0] )<categorify>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
def get_test_transforms(mode):
if mode == 0:
return A.Compose([
A.Normalize(always_apply=True),
ToTensorV2(p=1.0),
], p=1.0)
 elif mode == 1:
return A.Compose([
A.HorizontalFlip(p=1),
A.Normalize(always_apply=True),
ToTensorV2(p=1.0),
], p=1.0)
 elif mode == 2:
return A.Compose([
A.VerticalFlip(p=1),
A.Normalize(always_apply=True),
ToTensorV2(p=1.0),
], p=1.0)
 elif mode == 3:
return A.Compose([
A.InvertImg(p=1),
A.Normalize(always_apply=True),
ToTensorV2(p=1.0),
], p=1.0)
 else:
return A.Compose([
A.HorizontalFlip(p=1),
A.VerticalFlip(p=1),
A.Normalize(always_apply=True),
ToTensorV2(p=1.0),
], p=1.0 )<categorify>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
class DatasetSubmissionRetriever(Dataset):
def __init__(self, kinds, image_names, transforms=None):
super().__init__()
self.kinds = kinds
self.image_names = image_names
self.transforms = transforms
def __getitem__(self, index: int):
image_name = self.image_names[index]
image_kind = self.kinds[index]
image = lycon.load(f'{DATA_ROOT_PATH}/{image_kind}/{image_name}')
 if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image_name, image
def __len__(self)-> int:
return self.image_names.shape[0]<set_options>
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
del data_loader
del images
gc.collect()
torch.cuda.empty_cache()
!nvidia-smi<set_options>
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]
torch.cuda.empty_cache()<define_variables>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
results = []
test_imgs = glob('.. /input/alaska2-image-steganalysis/Test/*.jpg')
 for mode in range(0, 5):
dataset = DatasetSubmissionRetriever(
 image_names=np.array([path.split('/')[-1] for path in test_imgs]),
kinds=['Test']*len(test_imgs),
transforms=get_test_transforms(mode), 
)
 data_loader = DataLoader(
 dataset,
batch_size=26,
shuffle=False,
num_workers=4,
drop_last=False, 
)
 result = {'Id': [], 'Label': []}
for step,(image_names, images)in enumerate(data_loader):
torch.cuda.empty_cache()
print(step, end='\r')
 y_pred = net(images.cuda())[0]
y_pred = 1 - nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,0]
result['Id'].extend(image_names)
 result['Label'].extend(y_pred)
 results.append(result)
 print('done with mode', mode )<save_to_csv>
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 num_features
submissions = []
for mode in range(0,5):
submission = pd.DataFrame(results[mode])
 submissions.append(submission)
 submissions[mode].to_csv(f'submission_{mode}.csv', index=False )<save_to_csv>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
submissions[0]['Label'] =(
 submissions[0]['Label'] /3 +
submissions[1]['Label'] /6 +
submissions[2]['Label'] /6 +
submissions[4]['Label'] /6 +
submissions[3]['Label'] /12 
)
 submissions[0].to_csv(f'submission.csv', index=False)
 submissions[0]['Label'].hist(bins=100)
 plt.show()
submissions[0].head()<install_modules>
model = Model(num_features, num_all_targets)
 model
!pip install -q efficientnet_pytorch > /dev/null<set_options>
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in SEEDS:
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = [3, 4, 5, 6]
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<load_from_csv>
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols].values, train_df[target_cols_now].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols_now].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 oof = np.zeros(( len(train), len(target_cols_now)))
 best_loss = np.inf
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed_id}, FOLD: {fold_id}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"{tag_name}_FOLD{fold_id}_.pth")
 return oof
fine_tune_scheduler = FineTuneScheduler(EPOCHS)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.to(DEVICE)
 train_model(pretrained_model, 'ALL_TARGETS', all_target_cols)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.load_state_dict(torch.load(f"ALL_TARGETS_FOLD{fold_id}_.pth"))
pretrained_model.to(DEVICE)
 final_model = fine_tune_scheduler.copy_without_top(pretrained_model, num_features, num_all_targets, num_targets)
 oof = train_model(final_model, 'SCORED_ONLY', target_cols, fine_tune_scheduler)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f"SCORED_ONLY_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
<normalization>
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
def get_train_transforms() :
return A.Compose([
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0)
 def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<categorify>
SEED = [3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions
DATA_ROOT_PATH = '.. /input/alaska2-image-steganalysis'
def onehot(size, target):
vec = torch.zeros(size, dtype=torch.float32)
 vec[target] = 1.
 return vec
class DatasetRetriever(Dataset):
def __init__(self, kinds, image_names, labels, transforms=None):
super().__init__()
self.kinds = kinds
self.image_names = image_names
self.labels = labels
self.transforms = transforms
def __getitem__(self, index: int):
kind, image_name, label = self.kinds[index], self.image_names[index], self.labels[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{kind}/{image_name}', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
target = onehot(4, label)
 return image, target
def __len__(self)-> int:
return self.image_names.shape[0]
def get_labels(self):
return list(self.labels )<define_variables>
str(timedelta(seconds=time_diff))
fold_number = 0
train_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] != fold_number].kind.values,
image_names=dataset[dataset['fold'] != fold_number].image_name.values,
labels=dataset[dataset['fold'] != fold_number].label.values,
transforms=get_train_transforms() , 
)
 validation_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] == fold_number].kind.values,
image_names=dataset[dataset['fold'] == fold_number].image_name.values,
labels=dataset[dataset['fold'] == fold_number].label.values,
transforms=get_valid_transforms() ,
)<compute_train_metric>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
print(type(valid_results))
print(valid_results.shape)
 valid_results.to_csv('sub1.csv', index=False)
 train_targets_scored.to_csv('sub1_label.csv', index=False)
 score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def alaska_weighted_auc(y_true, y_valid):

 tpr_thresholds = [0.0, 0.4, 1.0]
weights = [2, 1]
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)
 areas = np.array(tpr_thresholds[1:])- np.array(tpr_thresholds[:-1])
 normalization = np.dot(areas, weights)
 competition_metric = 0
for idx, weight in enumerate(weights):
y_min = tpr_thresholds[idx]
y_max = tpr_thresholds[idx + 1]
mask =(y_min < tpr)&(tpr < y_max)
 x_padding = np.linspace(fpr[mask][-1], 1, 100)
 x = np.concatenate([fpr[mask], x_padding])
 y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])
 y = y - y_min
score = metrics.auc(x, y)
 submetric = score * weight
best_subscore =(y_max - y_min)* weight
competition_metric += submetric
return competition_metric / normalization
class RocAucMeter(object):
def __init__(self):
self.reset()
def reset(self):
self.y_true = np.array([0,1])
 self.y_pred = np.array([0.5,0.5])
 self.score = 0
def update(self, y_true, y_pred):
y_true = y_true.cpu().numpy().argmax(axis=1 ).clip(min=0, max=1 ).astype(int)
 y_pred = 1 - nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,0]
self.y_true = np.hstack(( self.y_true, y_true))
self.y_pred = np.hstack(( self.y_pred, y_pred))
self.score = alaska_weighted_auc(self.y_true, self.y_pred)
 @property
def avg(self):
return self.score<normalization>
sub1 = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0 )
class LabelSmoothing(nn.Module):
def __init__(self, smoothing = 0.05):
super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
def forward(self, x, target):
if self.training:
x = x.float()
target = target.float()
logprobs = torch.nn.functional.log_softmax(x, dim = -1)
 nll_loss = -logprobs * target
nll_loss = nll_loss.sum(-1)
 smooth_loss = -logprobs.mean(dim=-1)
 loss = self.confidence * nll_loss + self.smoothing * smooth_loss
return loss.mean()
else:
return torch.nn.functional.cross_entropy(x, target )<init_hyperparams>
! pip install.. /input/pytorch16gpu/torch-1.6.0cu101-cp37-cp37m-linux_x86_64.whl
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = './'
self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.criterion = LabelSmoothing().to(self.device)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss, final_scores = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint.bin')
 t = time.time()
summary_loss, final_scores = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch ).zfill(3)}epoch.bin')
 for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin')) [:-3]:
os.remove(path)
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(images, targets)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
targets = targets.to(self.device ).float()
batch_size = images.shape[0]
images = images.to(self.device ).float()
outputs = self.model(images)
 loss = self.criterion(outputs, targets)
 final_scores.update(targets, outputs)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss, final_scores
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(images, targets)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 targets = targets.to(self.device ).float()
images = images.to(self.device ).float()
batch_size = images.shape[0]
self.optimizer.zero_grad()
outputs = self.model(images)
 loss = self.criterion(outputs, targets)
 loss.backward()
final_scores.update(targets, outputs)
 summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return summary_loss, final_scores
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )<choose_model_class>
!pip install.. /input/iterative-stratification-tabnet/iterative_stratification-0.1.6-py3-none-any.whl
def get_net() :
net = EfficientNet.from_pretrained('efficientnet-b2')
 net._fc = nn.Linear(in_features=1408, out_features=4, bias=True)
 return net
net = get_net().cuda()<init_hyperparams>
os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
class TrainGlobalConfig:
num_workers = 4
batch_size = 16
n_epochs = 25
lr = 0.001
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08 
)
<load_pretrained>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42)
 n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
def run_training() :
device = torch.device('cuda:0')
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
sampler=BalanceClassSampler(labels=train_dataset.get_labels() , mode="downsampling"),
batch_size=TrainGlobalConfig.batch_size,
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, val_loader )<load_from_csv>
def get_ratio_labels(df):
columns = list(df.columns)
 columns.pop(0)
 ratios = []
toremove = []
for c in columns:
counts = df[c].value_counts()
if len(counts)!= 1:
ratios.append(counts[0]/counts[1])
 else:
toremove.append(c)
 print(f"remove {len(toremove)} columns")
 for t in toremove:
columns.remove(t)
 return columns, np.array(ratios ).astype(np.int32)
 columns, ratios = get_ratio_labels(train_targets_scored)
 columns_nonscored, ratios_nonscored = get_ratio_labels(train_targets_nonscored )
file = open('.. /input/alaska2-public-baseline/log.txt', 'r')
 for line in file.readlines() :
print(line[:-1])
 file.close()<load_pretrained>
remove_vehicle=True
def transform_data(train, test, col, normalize=True, removed_vehicle=False):

 mapping = {"cp_type":{"trt_cp": 0, "ctl_vehicle":1},
"cp_time":{48:0, 72:1, 24:2},
"cp_dose":{"D1":0, "D2":1}}
if removed_vehicle:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 else:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 max_ = 10.
 min_ = -10.
 if removed_vehicle:
numerical_tr = train[col[3:]].values
numerical_test = test[col[3:]].values
else:
numerical_tr = train[col[3:]].values
numerical_test = test[col[3:]].values
if normalize:
numerical_tr =(numerical_tr-min_)/(max_ - min_)
 numerical_test =(numerical_test-min_)/(max_ - min_)
 return categories_tr, categories_test, numerical_tr, numerical_test
col_features = list(train_features.columns)[1:]
cat_tr, cat_test, numerical_tr, numerical_test = transform_data(train_features, test_features, col_features, normalize=False, removed_vehicle=remove_vehicle)
 targets_tr = train_targets_scored[columns].values.astype(np.float32)
 targets2_tr = train_targets_nonscored[columns_nonscored].values.astype(np.float32 )
checkpoint = torch.load('.. /input/alaska2-public-baseline/best-checkpoint-033epoch.bin')
 net.load_state_dict(checkpoint['model_state_dict']);
net.eval() ;<data_type_conversions>
def inference_fn(model, X ,verbose=True):
with torch.no_grad() :
y_preds = model.predict(X)
 y_preds = torch.sigmoid(torch.as_tensor(y_preds)).numpy()
return y_preds
class DatasetSubmissionRetriever(Dataset):
def __init__(self, image_names, transforms=None):
super().__init__()
self.image_names = image_names
self.transforms = transforms
def __getitem__(self, index: int):
image_name = self.image_names[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/Test/{image_name}', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image_name, image
def __len__(self)-> int:
return self.image_names.shape[0]<define_variables>
def log_loss_score(actual, predicted, eps=1e-15):

 p1 = actual * np.log(predicted+eps)
 p0 =(1-actual)* np.log(1-predicted+eps)
 loss = p0 + p1
return -loss.mean()
dataset = DatasetSubmissionRetriever(
 image_names=np.array([path.split('/')[-1] for path in glob('.. /input/alaska2-image-steganalysis/Test/*.jpg')]),
transforms=get_valid_transforms() , 
)
 data_loader = DataLoader(
 dataset,
batch_size=8,
shuffle=False,
num_workers=2,
drop_last=False,
)<save_to_csv>
def log_loss_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
results[i] = log_loss_score(y_true[:,i], y_pred[:,i])
 return results.mean()
submission = pd.DataFrame(result)
 submission.to_csv('submission.csv', index=False)
 submission.head()<import_modules>
def auc_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
try:
results[i] = roc_auc_score(y_true[:,i], y_pred[:,i])
 except:
pass
return results.mean()
!pip install -q efficientnet_pytorch
Compose, HorizontalFlip, CLAHE, HueSaturationValue,
RandomBrightness, RandomContrast, RandomGamma, OneOf, Resize,
ToFloat, ShiftScaleRotate, GridDistortion, ElasticTransform, JpegCompression, HueSaturationValue,
RGBShift, RandomBrightness, RandomContrast, Blur, MotionBlur, MedianBlur, GaussNoise, CenterCrop,
IAAAdditiveGaussianNoise, GaussNoise, OpticalDistortion, RandomSizedCrop, VerticalFlip 
)
<set_options>
create_dataloaders,
create_explain_matrix)
 class TabModel(BaseEstimator):
def __init__(self, n_d=8, n_a=8, n_steps=3, gamma=1.3, cat_idxs=[], cat_dims=[], cat_emb_dim=1,
n_independent=2, n_shared=2, epsilon=1e-15, momentum=0.02,
lambda_sparse=1e-3, seed=0,
clip_value=1, verbose=1,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2),
scheduler_params=None, scheduler_fn=None,
mask_type="sparsemax",
input_dim=None, output_dim=None,
device_name='auto'):

 self.n_d = n_d
self.n_a = n_a
self.n_steps = n_steps
self.gamma = gamma
self.cat_idxs = cat_idxs
self.cat_dims = cat_dims
self.cat_emb_dim = cat_emb_dim
self.n_independent = n_independent
self.n_shared = n_shared
self.epsilon = epsilon
self.momentum = momentum
self.lambda_sparse = lambda_sparse
self.clip_value = clip_value
self.verbose = verbose
self.optimizer_fn = optimizer_fn
self.optimizer_params = optimizer_params
self.device_name = device_name
self.scheduler_params = scheduler_params
self.scheduler_fn = scheduler_fn
self.mask_type = mask_type
self.input_dim = input_dim
self.output_dim = output_dim
self.batch_size = 1024
self.seed = seed
torch.manual_seed(self.seed)
 if device_name == 'auto':
if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
self.device = torch.device(device_name)
 print(f"Device used : {self.device}")
 @abstractmethod
def construct_loaders(self, X_train, y_train, X_valid, y_valid,
weights, batch_size, num_workers, drop_last):

 raise NotImplementedError('users must define construct_loaders to use this base class')
 def init_network(
 self,
input_dim,
output_dim,
n_d,
n_a,
n_steps,
gamma,
cat_idxs,
cat_dims,
cat_emb_dim,
n_independent,
n_shared,
epsilon,
virtual_batch_size,
momentum,
device_name,
mask_type, 
):
self.network = tab_network.TabNet(
 input_dim,
output_dim,
n_d=n_d,
n_a=n_a,
n_steps=n_steps,
gamma=gamma,
cat_idxs=cat_idxs,
cat_dims=cat_dims,
cat_emb_dim=cat_emb_dim,
n_independent=n_independent,
n_shared=n_shared,
epsilon=epsilon,
virtual_batch_size=virtual_batch_size,
momentum=momentum,
device_name=device_name,
mask_type=mask_type ).to(self.device)
 self.reducing_matrix = create_explain_matrix(
 self.network.input_dim,
self.network.cat_emb_dim,
self.network.cat_idxs,
self.network.post_embed_dim)
 def fit(self, X_train, y_train, X_valid=None, y_valid=None, loss_fn=None,
weights=0, max_epochs=100, patience=10, batch_size=1024,
virtual_batch_size=128, num_workers=0, drop_last=False):

 self.update_fit_params(X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience, batch_size,
virtual_batch_size, num_workers, drop_last)
 train_dataloader, valid_dataloader = self.construct_loaders(X_train,
y_train,
X_valid,
y_valid,
self.updated_weights,
self.batch_size,
self.num_workers,
self.drop_last)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=self.virtual_batch_size,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.optimizer = self.optimizer_fn(self.network.parameters() ,
**self.optimizer_params)
 if self.scheduler_fn:
self.scheduler = self.scheduler_fn(self.optimizer, **self.scheduler_params)
 else:
self.scheduler = None
self.losses_train = []
self.losses_valid = []
self.learning_rates = []
self.metrics_train = []
self.metrics_valid = []
if self.verbose > 0:
print("Will train until validation stopping metric",
f"hasn't improved in {self.patience} rounds.")
 msg_epoch = f'| EPOCH | train | valid | total time(s)'
print('---------------------------------------')
 print(msg_epoch)
 total_time = 0
while(self.epoch < self.max_epochs and
self.patience_counter < self.patience):
starting_time = time.time()
self.learning_rates.append(self.optimizer.param_groups[-1]["lr"])
 fit_metrics = self.fit_epoch(train_dataloader, valid_dataloader)
 self.losses_train.append(fit_metrics['train']['loss_avg'])
 self.losses_valid.append(fit_metrics['valid']['total_loss'])
 self.metrics_train.append(fit_metrics['train']['stopping_loss'])
 self.metrics_valid.append(fit_metrics['valid']['stopping_loss'])
 stopping_loss = fit_metrics['valid']['stopping_loss']
if stopping_loss < self.best_cost:
self.best_cost = stopping_loss
self.patience_counter = 0
self.best_network = deepcopy(self.network)
 has_improved = True
else:
self.patience_counter += 1
has_improved=False
self.epoch += 1
total_time += time.time() - starting_time
if self.verbose > 0:
if self.epoch % self.verbose == 0:
separator = "|"
msg_epoch = f"| {self.epoch:<5} | "
msg_epoch += f" {fit_metrics['train']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {fit_metrics['valid']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {np.round(total_time, 1):<10}"
msg_epoch += f" {has_improved}"
print(msg_epoch)
 if self.verbose > 0:
if self.patience_counter == self.patience:
print(f"Early stopping occured at epoch {self.epoch}")
 print(f"Training done in {total_time:.3f} seconds.")
 print('---------------------------------------')
 self.history = {"train": {"loss": self.losses_train,
"metric": self.metrics_train,
"lr": self.learning_rates},
"valid": {"loss": self.losses_valid,
"metric": self.metrics_valid}}
self.load_best_model()
self._compute_feature_importances(train_dataloader)
 def save_model(self, path):

 saved_params = {}
for key, val in self.get_params().items() :
if isinstance(val, type):
continue
else:
saved_params[key] = val
Path(path ).mkdir(parents=True, exist_ok=True)
 with open(Path(path ).joinpath("model_params.json"), "w", encoding="utf8")as f:
json.dump(saved_params, f)
 torch.save(self.network.state_dict() , Path(path ).joinpath("network.pt"))
shutil.make_archive(path, 'zip', path)
 shutil.rmtree(path)
 print(f"Successfully saved model at {path}.zip")
 return f"{path}.zip"
def load_model(self, filepath):
try:
try:
with zipfile.ZipFile(filepath)as z:
with z.open("model_params.json")as f:
loaded_params = json.load(f)
 with z.open("network.pt")as f:
try:
saved_state_dict = torch.load(f)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.BytesIO(f.read()))
except:
with open(os.path.join(filepath, "model_params.json")) as f:
loaded_params = json.load(f)
 saved_state_dict = torch.load(os.path.join(filepath, "network.pt"), map_location="cpu")
 except KeyError:
raise KeyError("Your zip file is missing at least one component")
 if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
loaded_params["device_name"] = device_name
self.__init__(**loaded_params)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=1024,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.network.load_state_dict(saved_state_dict)
 self.network.eval()
return
def fit_epoch(self, train_dataloader, valid_dataloader):

 train_metrics = self.train_epoch(train_dataloader)
 valid_metrics = self.predict_epoch(valid_dataloader)
 fit_metrics = {'train': train_metrics,
'valid': valid_metrics}
return fit_metrics
@abstractmethod
def train_epoch(self, train_loader):

 raise NotImplementedError('users must define train_epoch to use this base class')
 @abstractmethod
def train_batch(self, data, targets):

 raise NotImplementedError('users must define train_batch to use this base class')
 @abstractmethod
def predict_epoch(self, loader):

 raise NotImplementedError('users must define predict_epoch to use this base class')
 @abstractmethod
def predict_batch(self, data, targets):

 raise NotImplementedError('users must define predict_batch to use this base class')
 def load_best_model(self):
if self.best_network is not None:
self.network = self.best_network
@abstractmethod
def predict(self, X):

 raise NotImplementedError('users must define predict to use this base class')
 def explain(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 for key, value in masks.items() :
masks[key] = csc_matrix.dot(value.cpu().detach().numpy() ,
self.reducing_matrix)
 if batch_nb == 0:
res_explain = csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)
 res_masks = masks
else:
res_explain = np.vstack([res_explain,
csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)])
 for key, value in masks.items() :
res_masks[key] = np.vstack([res_masks[key], value])
 return res_explain, res_masks
def _compute_feature_importances(self, loader):
self.network.eval()
feature_importances_ = np.zeros(( self.network.post_embed_dim))
for data, targets in loader:
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 feature_importances_ += M_explain.sum(dim=0 ).cpu().detach().numpy()
feature_importances_ = csc_matrix.dot(feature_importances_,
self.reducing_matrix)
 self.feature_importances_ = feature_importances_ / np.sum(feature_importances_)
 class TabNetRegressor(TabModel):
def construct_loaders(self, X_train, y_train, X_valid, y_valid, weights,
batch_size, num_workers, drop_last):

 if isinstance(weights, int):
if weights == 1:
raise ValueError("Please provide a list of weights for regression.")
 if isinstance(weights, dict):
raise ValueError("Please provide a list of weights for regression.")
 train_dataloader, valid_dataloader = create_dataloaders(X_train,
y_train,
X_valid,
y_valid,
weights,
batch_size,
num_workers,
drop_last)
 return train_dataloader, valid_dataloader
def update_fit_params(self, X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience,
batch_size, virtual_batch_size, num_workers, drop_last):
if loss_fn is None:
self.loss_fn = torch.nn.functional.mse_loss
else:
self.loss_fn = loss_fn
assert X_train.shape[1] == X_valid.shape[1], "Dimension mismatch X_train X_valid"
self.input_dim = X_train.shape[1]
if len(y_train.shape)== 1:
raise ValueError()
assert y_train.shape[1] == y_valid.shape[1], "Dimension mismatch y_train y_valid"
self.output_dim = y_train.shape[1]
self.updated_weights = weights
self.max_epochs = max_epochs
self.patience = patience
self.batch_size = batch_size
self.virtual_batch_size = virtual_batch_size
self.patience_counter = 0
self.epoch = 0
self.best_cost = np.inf
self.num_workers = num_workers
self.drop_last = drop_last
def train_epoch(self, train_loader):

 self.network.train()
y_preds = []
ys = []
total_loss = 0
for data, targets in train_loader:
batch_outs = self.train_batch(data, targets)
 y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 total_loss += batch_outs["loss"]
y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(train_loader)
 epoch_metrics = {'loss_avg': total_loss,
'stopping_loss': total_loss,
}
if self.scheduler is not None:
self.scheduler.step()
return epoch_metrics
def train_batch(self, data, targets):

 self.network.train()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
self.optimizer.zero_grad()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss.backward()
if self.clip_value:
clip_grad_norm_(self.network.parameters() , self.clip_value)
 self.optimizer.step()
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict_epoch(self, loader):

 y_preds = []
ys = []
self.network.eval()
total_loss = 0
for data, targets in loader:
batch_outs = self.predict_batch(data, targets)
 total_loss += batch_outs["loss"]
y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(loader)
 epoch_metrics = {'total_loss': total_loss,
'stopping_loss': stopping_loss}
return epoch_metrics
def predict_batch(self, data, targets):

 self.network.eval()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 results = []
for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
output, M_loss = self.network(data)
 predictions = output.cpu().detach().numpy()
results.append(predictions)
 res = np.vstack(results)
 return res
seed = 42
print(f'setting everything to seed {seed}')
 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<define_variables>
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold, MultilabelStratifiedShuffleSplit
from sklearn.metrics import roc_auc_score
data_dir = '.. /input/alaska2-image-steganalysis'
folder_names = ['JMiPOD/', 'JUNIWARD/', 'UERD/']
class_names = ['Normal', 'JMiPOD_75', 'JMiPOD_90', 'JMiPOD_95',
'JUNIWARD_75', 'JUNIWARD_90', 'JUNIWARD_95',
'UERD_75', 'UERD_90', 'UERD_95']
class_labels = { name: i for i, name in enumerate(class_names)}<load_from_csv>
class Config(object):
def __init__(self):
self.num_class = targets_tr.shape[1]
self.verbose=False
self.device = "cuda" if torch.cuda.is_available() else "cpu"
self.SPLITS = 10
self.EPOCHS = 200
self.num_ensembling = 1
self.seed = 0
self.cat_emb_dim=[1] * cat_tr.shape[1]
self.cats_idx = list(range(cat_tr.shape[1]))
self.cat_dims = [len(np.unique(cat_tr[:, i])) for i in range(cat_tr.shape[1])]
self.num_numericals= numerical_tr.shape[1]
self.save_name = ".. /input/tabnet-smoothing/tabnet_raw_step1"
self.strategy = "KFOLD"
cfg = Config()
train_df = pd.read_csv('.. /input/alaska2trainvalsplit/alaska2_train_df.csv')
 val_df = pd.read_csv('.. /input/alaska2trainvalsplit/alaska2_val_df.csv')
 print(train_df.sample(10))
train_df.Label.hist()
plt.title('Distribution of Classes' )<normalization>
X_test = np.concatenate([cat_test, numerical_test ], axis=1 )
class Alaska2Dataset(Dataset):
def __init__(self, df, augmentations=None):
self.data = df
self.augment = augmentations
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
fn, label = self.data.loc[idx]
im = cv2.imread(fn)[:, :, ::-1]
if self.augment:
im = self.augment(image=im)
 return im, label
img_size = 512
AUGMENTATIONS_TRAIN = Compose([
VerticalFlip(p=0.5),
HorizontalFlip(p=0.5),
ToFloat(max_value=255),
ToTensor()
], p=1)
 AUGMENTATIONS_TEST = Compose([
ToFloat(max_value=255),
ToTensor()
], p=1)
<create_dataframe>
SEED = [0,1,2,3,4,5,6]
if cfg.strategy == "KFOLD":
oof_preds_all = []
oof_targets_all = []
scores_all = []
scores_auc_all= []
preds_test = []
for seed in SEED:
print("
mskf = MultilabelStratifiedKFold(n_splits=5, random_state=cfg.seed+seed, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
p = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 X_train, y_train = torch.as_tensor(np.concatenate([cat_tr[train_idx], numerical_tr[train_idx] ], axis=1)) , torch.as_tensor(targets_tr[train_idx])
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx] ], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, gamma=1.3, lambda_sparse=0, cat_dims=cfg.cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cfg.cats_idx, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2), mask_type='entmax', device_name=cfg.device, scheduler_params=dict(milestones=[ 50,100,150], gamma=0.9), scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 name = cfg.save_name + f"_fold{j}_{seed}"
model.load_model(name)
 preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 p = np.stack(p)
 preds_test.append(p)
 oof_preds_all.append(np.concatenate(oof_preds))
oof_targets_all.append(np.concatenate(oof_targets))
scores_all.append(np.array(scores))
scores_auc_all.append(np.array(scores_auc))
preds_test = np.stack(preds_test )
temp_df = train_df.sample(64 ).reset_index(drop=True)
 train_dataset = Alaska2Dataset(temp_df, augmentations=AUGMENTATIONS_TEST)
 batch_size = 64
num_workers = 0
temp_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
num_workers=num_workers, shuffle=False)
 images, labels = next(iter(temp_loader))
images = images['image'].permute(0, 2, 3, 1)
 max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width,
figsize=(grid_width+1, grid_height+1))
for i,(im, label)in enumerate(zip(images, labels)) :
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im.squeeze())
 ax.set_title(str(label.item()))
ax.axis('off')
 plt.suptitle("0: No Hidden Message, 1: JMiPOD, 2: JUNIWARD, 3:UERD")
 plt.show()
del images
gc.collect()<create_dataframe>
if cfg.strategy == "KFOLD":
pred_mean = np.stack(oof_preds_all ).mean(0)
 print("pred_mean: ", pred_mean.shape)
 np.save("sub2.npy", pred_mean)
 target_mean = np.stack(oof_targets_all ).mean(0)
 print("target_mean: ", target_mean.shape)
 np.save("sub2_label.npy", target_mean)
 for i in range(cfg.num_ensembling):
print("CV score fold : ", log_loss_multi(oof_targets_all[i], oof_preds_all[i]))
print("auc mean : ", sum(scores_auc_all[i])/len(scores_auc_all[i]))
train_dataset = Alaska2Dataset(temp_df, augmentations=AUGMENTATIONS_TRAIN)
 batch_size = 64
num_workers = 0
temp_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
num_workers=num_workers, shuffle=False)
 images, labels = next(iter(temp_loader))
images = images['image'].permute(0, 2, 3, 1)
 max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width,
figsize=(grid_width+1, grid_height+1))
for i,(im, label)in enumerate(zip(images, labels)) :
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im.squeeze())
 ax.set_title(str(label.item()))
ax.axis('off')
 plt.suptitle("0: No Hidden Message, 1: JMiPOD, 2: JUNIWARD, 3:UERD")
 plt.show()
del images, temp_df
gc.collect()<categorify>
submission[columns] = preds_test.mean(1 ).mean(0)
 submission.loc[test_features['cp_type']=='ctl_vehicle', submission.columns[1:]] = 0
class Net(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.model = EfficientNet.from_pretrained('efficientnet-b0')
 self.dense_output = nn.Linear(1280, num_classes)
 def forward(self, x):
feat = self.model.extract_features(x)
 feat = F.avg_pool2d(feat, feat.size() [2:] ).reshape(-1, 1280)
 return self.dense_output(feat )<create_dataframe>
sub2 = submission
batch_size = 8
num_workers = 8
train_dataset = Alaska2Dataset(train_df, augmentations=AUGMENTATIONS_TRAIN)
 valid_dataset = Alaska2Dataset(val_df.sample(1000 ).reset_index(drop=True), augmentations=AUGMENTATIONS_TEST)
 train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
num_workers=num_workers,
shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=batch_size*2,
num_workers=num_workers,
shuffle=False)
 device = 'cuda'
model = Net(num_classes=len(class_labels)).to(device)
 model.load_state_dict(torch.load('.. /input/alaska-model-weights/epoch_4_val_loss_7.0_auc_0.831.pth'))
dones = 5
optimizer = torch.optim.AdamW(model.parameters() , lr=1e-4 )<compute_train_metric>
FOLDS = 10
EPOCHS = 80
BATCH_SIZE = 124
LR = 0.001
VERBOSE = 0
SEEDS1 = [1, 2, 3, 4, 5, 6, 7]
SEEDS2 = [8, 9, 10, 11, 12, 13, 14]
SEEDS3 = [15, 16, 17, 18, 19, 20, 21]
SEEDS4 = [22, 23, 24, 25, 26, 27, 28]
SEEDS5 = [29, 30, 31, 32, 33, 34, 35]
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
def alaska_weighted_auc(y_true, y_valid):
tpr_thresholds = [0.0, 0.4, 1.0]
weights = [2, 1]
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)
 areas = np.array(tpr_thresholds[1:])- np.array(tpr_thresholds[:-1])
 normalization = np.dot(areas, weights)
 competition_metric = 0
for idx, weight in enumerate(weights):
y_min = tpr_thresholds[idx]
y_max = tpr_thresholds[idx + 1]
mask =(y_min < tpr)&(tpr < y_max)
 x_padding = np.linspace(fpr[mask][-1], 1, 100)
 x = np.concatenate([fpr[mask], x_padding])
 y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])
 y = y - y_min
score = metrics.auc(x, y)
 submetric = score * weight
best_subscore =(y_max - y_min)* weight
competition_metric += submetric
return competition_metric / normalization<train_on_grid>
def mapping_and_filter(train, train_targets, test):
cp_type = {'trt_cp': 0, 'ctl_vehicle': 1}
cp_dose = {'D1': 0, 'D2': 1}
for df in [train, test]:
df['cp_type'] = df['cp_type'].map(cp_type)
 df['cp_dose'] = df['cp_dose'].map(cp_dose)
 train_targets = train_targets[train['cp_type'] == 0].reset_index(drop = True)
 train = train[train['cp_type'] == 0].reset_index(drop = True)
 train_targets.drop(['sig_id'], inplace = True, axis = 1)
 return train, train_targets, test
def scaling(train, test):
features = train.columns[2:]
scaler = RobustScaler()
scaler.fit(pd.concat([train[features], test[features]], axis = 0))
train[features] = scaler.transform(train[features])
 test[features] = scaler.transform(test[features])
 return train, test, features
def fe_pca(train, test, n_components_g = 70, n_components_c = 10, SEED = 123):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_pca(train, test, features, kind = 'g', n_components = n_components_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 pca = PCA(n_components = n_components, random_state = SEED)
 data = pca.fit_transform(data)
 columns = [f'pca_{kind}{i + 1}' for i in range(n_components)]
data = pd.DataFrame(data, columns = columns)
 train_ = data.iloc[:train.shape[0]]
test_ = data.iloc[train.shape[0]:].reset_index(drop = True)
 train = pd.concat([train, train_], axis = 1)
 test = pd.concat([test, test_], axis = 1)
 return train, test
train, test = create_pca(train, test, features_g, kind = 'g', n_components = n_components_g)
 train, test = create_pca(train, test, features_c, kind = 'c', n_components = n_components_c)
 return train, test
def fe_stats(train, test):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 for df in [train, test]:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train, test
def c_squared(train, test):
features_c = list(train.columns[776:876])
 for df in [train, test]:
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
def mean_log_loss(y_true, y_pred):
y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
 metrics = []
for target in range(206):
metrics.append(log_loss(y_true[:, target], y_pred[:, target]))
return np.mean(metrics)
 def create_model_rs(shape1, shape2):
input_1 = tf.keras.layers.Input(shape =(shape1))
input_2 = tf.keras.layers.Input(shape =(shape2))
head_1 = tf.keras.layers.BatchNormalization()(input_1)
 head_1 = tf.keras.layers.Dropout(0.2 )(head_1)
 head_1 = tf.keras.layers.Dense(512, activation = "elu" )(head_1)
 head_1 = tf.keras.layers.BatchNormalization()(head_1)
 input_3 = tf.keras.layers.Dense(256, activation = "elu" )(head_1)
 input_3_concat = tf.keras.layers.Concatenate()([input_2, input_3])
 head_2 = tf.keras.layers.BatchNormalization()(input_3_concat)
 head_2 = tf.keras.layers.Dropout(0.3 )(head_2)
 head_2 = tf.keras.layers.Dense(512, "relu" )(head_2)
 head_2 = tf.keras.layers.BatchNormalization()(head_2)
 head_2 = tf.keras.layers.Dense(512, "elu" )(head_2)
 head_2 = tf.keras.layers.BatchNormalization()(head_2)
 head_2 = tf.keras.layers.Dense(256, "relu" )(head_2)
 head_2 = tf.keras.layers.BatchNormalization()(head_2)
 input_4 = tf.keras.layers.Dense(256, "elu" )(head_2)
 input_4_avg = tf.keras.layers.Average()([input_3, input_4])
 head_3 = tf.keras.layers.BatchNormalization()(input_4_avg)
 head_3 = tf.keras.layers.Dense(256, kernel_initializer = 'lecun_normal', activation = 'selu' )(head_3)
 head_3 = tf.keras.layers.BatchNormalization()(head_3)
 head_3 = tf.keras.layers.Dense(206, kernel_initializer = 'lecun_normal', activation = 'selu' )(head_3)
 head_3 = tf.keras.layers.BatchNormalization()(head_3)
 output = tf.keras.layers.Dense(206, activation = "sigmoid" )(head_3)
 model = tf.keras.models.Model(inputs = [input_1, input_2], outputs = output)
 opt = tf.optimizers.Adam(learning_rate = LR)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0015),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def create_model_5l(shape):
inp = tf.keras.layers.Input(shape =(shape))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(2560, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(2048, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1524, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1012, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(780, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(206, activation = 'sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 opt = tf.optimizers.Adam(learning_rate = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0020),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def create_model_4l(shape):
inp = tf.keras.layers.Input(shape =(shape))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(2048, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1524, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1012, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1012, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(206, activation = 'sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 opt = tf.optimizers.Adam(learning_rate = LR)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0020),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def create_model_3l(shape):
inp = tf.keras.layers.Input(shape =(shape))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(0.4914099166744246 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(1159, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.18817607797795838 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(960, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.12542057776853896 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(1811, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.20175242230280122 )(x)
 out = tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation = 'sigmoid'))(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 opt = tf.optimizers.Adam(learning_rate = LR)
 opt = tfa.optimizers.Lookahead(opt, sync_period = 10)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0015),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def create_model_2l(shape):
inp = tf.keras.layers.Input(shape =(shape))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(0.2688628097505064 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(1292, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4598218403250696 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(983, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4703144018483698 )(x)
 out = tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation = 'sigmoid'))(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 opt = tf.optimizers.Adam(learning_rate = LR)
 opt = tfa.optimizers.Lookahead(opt, sync_period = 10)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0015),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def train_and_evaluate(train, test, train_targets, features, start_predictors, SEED = 123, MODEL = '3l'):
seed_everything(SEED)
 oof_pred = np.zeros(( train.shape[0], 206))
test_pred = np.zeros(( test.shape[0], 206))
for fold,(trn_ind, val_ind)in enumerate(MultilabelStratifiedKFold(n_splits = FOLDS,
random_state = SEED,
shuffle = True)\ 
.split(train_targets, train_targets)) :
K.clear_session()
if MODEL == '5l':
model = create_model_5l(len(features))
elif MODEL == '4l':
model = create_model_4l(len(features))
elif MODEL == '3l':
model = create_model_3l(len(features))
elif MODEL == '2l':
model = create_model_2l(len(features))
elif MODEL == "rs":
model = create_model_rs(len(features), len(start_predictors))
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_binary_crossentropy',
mode = 'min',
patience = 10,
restore_best_weights = False,
verbose = VERBOSE)
 reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_binary_crossentropy',
mode = 'min',
factor = 0.3,
patience = 3,
verbose = VERBOSE)
 checkpoint = tf.keras.callbacks.ModelCheckpoint(f'{MODEL}_{fold}_{SEED}.h5',
monitor = 'val_binary_crossentropy',
verbose = VERBOSE,
save_best_only = True,
save_weights_only = True)
 x_train, x_val = train[features].values[trn_ind], train[features].values[val_ind]
y_train, y_val = train_targets.values[trn_ind], train_targets.values[val_ind]
if MODEL == "rs":
x_train_, x_val_ = train[start_predictors].values[trn_ind], train[start_predictors].values[val_ind]
model.fit([x_train, x_train_], y_train,
validation_data =([x_val, x_val_], y_val),
epochs = EPOCHS,
batch_size = BATCH_SIZE,
callbacks = [early_stopping, reduce_lr, checkpoint],
verbose = VERBOSE)
 model.load_weights(f'{MODEL}_{fold}_{SEED}.h5')
 oof_pred[val_ind] = model.predict([x_val, x_val_])
 test_pred += model.predict([test[features].values, test[start_predictors].values])/ FOLDS
else:
model.fit(x_train, y_train,
validation_data =(x_val, y_val),
epochs = EPOCHS,
batch_size = BATCH_SIZE,
callbacks = [early_stopping, reduce_lr, checkpoint],
verbose = VERBOSE)
 model.load_weights(f'{MODEL}_{fold}_{SEED}.h5')
 oof_pred[val_ind] = model.predict(x_val)
 test_pred += model.predict(test[features].values)/ FOLDS
oof_score = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our out of folds mean log loss score is {oof_score}')
 return test_pred, oof_pred
def inference(train, test, train_targets, features, start_predictors, SEED = 123, MODEL = '3l', PATH = '.. /input/moa-3layer'):
seed_everything(SEED)
 oof_pred = np.zeros(( train.shape[0], 206))
test_pred = np.zeros(( test.shape[0], 206))
for fold,(trn_ind, val_ind)in enumerate(MultilabelStratifiedKFold(n_splits = FOLDS,
random_state = SEED,
shuffle = True)\ 
.split(train_targets, train_targets)) :
K.clear_session()
if MODEL == '5l':
model = create_model_5l(len(features))
elif MODEL == '4l':
model = create_model_4l(len(features))
elif MODEL == '3l':
model = create_model_3l(len(features))
elif MODEL == '2l':
model = create_model_2l(len(features))
elif MODEL == "rs":
model = create_model_rs(len(features), len(start_predictors))
x_train, x_val = train[features].values[trn_ind], train[features].values[val_ind]
y_train, y_val = train_targets.values[trn_ind], train_targets.values[val_ind]
model.load_weights('.. /input/moa-5seed/Model_Weights'+'/'+MODEL+'_'+str(fold)+'_'+str(SEED)+'.h5')
 if MODEL == "rs":
x_train_, x_val_ = train[start_predictors].values[trn_ind], train[start_predictors].values[val_ind]
oof_pred[val_ind] = model.predict([x_val, x_val_])
 test_pred += model.predict([test[features].values, test[start_predictors].values])/ FOLDS
else:
oof_pred[val_ind] = model.predict(x_val)
 test_pred += model.predict(test[features].values)/ FOLDS
oof_score = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our out of folds mean log loss score is {oof_score}')
 return test_pred, oof_pred
def run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = [123], MODEL = '3l', PATH = '.. /input/moa-3layer'):
test_pred = []
oof_pred = []
for SEED in SEEDS:
print(f'Using model {MODEL} with seed {SEED} for inference')
 print(f'Trained with {len(features)} features')
 test_pred_, oof_pred_ = inference(train, test, train_targets, features, start_predictors, SEED = SEED, MODEL = MODEL, PATH = PATH)
 test_pred.append(test_pred_)
 oof_pred.append(oof_pred_)
 print('-'*50)
 print('
')
 test_pred = np.average(test_pred, axis = 0)
 oof_pred = np.average(oof_pred, axis = 0)
 seed_log_loss = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our out of folds log loss for our seed blend model is {seed_log_loss}')
 return test_pred, oof_pred
def submission(test_pred):
sample_submission.loc[:, train_targets.columns] = test_pred
sample_submission.loc[test['cp_type'] == 1, train_targets.columns] = 0
sample_submission.to_csv('submission.csv', index = False)
 return sample_submission
start_predictors = ["g-0", "g-7", "g-8", "g-10", "g-13", "g-17", "g-20", "g-22", "g-24", "g-26", "g-28", "g-29", "g-30", "g-31", "g-32", "g-34", "g-35", "g-36", "g-37", "g-38",
"g-39","g-41", "g-46", "g-48", "g-50", "g-51", "g-52", "g-55", "g-58", "g-59", "g-61", "g-62", "g-63", "g-65", "g-66", "g-67", "g-68", "g-70", "g-72", "g-74",
"g-75", "g-79", "g-83", "g-84", "g-85", "g-86", "g-90", "g-91", "g-94", "g-95", "g-96", "g-97", "g-98", "g-100", "g-102", "g-105", "g-106", "g-112", "g-113",
"g-114", "g-116", "g-121", "g-123", "g-126", "g-128", "g-131", "g-132", "g-134", "g-135", "g-138", "g-139", "g-140", "g-142", "g-144", "g-145", "g-146",
"g-147", "g-148", "g-152", "g-155", "g-157", "g-158", "g-160", "g-163", "g-164", "g-165", "g-170", "g-173", "g-174", "g-175", "g-177", "g-178", "g-181",
"g-183", "g-185", "g-186", "g-189", "g-192", "g-194", "g-195", "g-196", "g-197", "g-199", "g-201", "g-202", "g-206", "g-208", "g-210", "g-213", "g-214",
"g-215", "g-220", "g-226", "g-228", "g-229", "g-235", "g-238", "g-241", "g-242", "g-243", "g-244", "g-245", "g-248", "g-250", "g-251", "g-254", "g-257",
"g-259", "g-261", "g-266", "g-270", "g-271", "g-272", "g-275", "g-278", "g-282", "g-287", "g-288", "g-289", "g-291", "g-293", "g-294", "g-297", "g-298",
"g-301", "g-303", "g-304", "g-306", "g-308", "g-309", "g-310", "g-311", "g-314", "g-315", "g-316", "g-317", "g-320", "g-321", "g-322", "g-327", "g-328",
"g-329", "g-332", "g-334", "g-335", "g-336", "g-337", "g-339", "g-342", "g-344", "g-349", "g-350", "g-351", "g-353", "g-354", "g-355", "g-357", "g-359",
"g-360", "g-364", "g-365", "g-366", "g-367", "g-368", "g-369", "g-374", "g-375", "g-377", "g-379", "g-385", "g-386", "g-390", "g-392", "g-393", "g-400",
"g-402", "g-406", "g-407", "g-409", "g-410", "g-411", "g-414", "g-417", "g-418", "g-421", "g-423", "g-424", "g-427", "g-429", "g-431", "g-432", "g-433",
"g-434", "g-437", "g-439", "g-440", "g-443", "g-449", "g-458", "g-459", "g-460", "g-461", "g-464", "g-467", "g-468", "g-470", "g-473", "g-477", "g-478",
"g-479", "g-484", "g-485", "g-486", "g-488", "g-489", "g-491", "g-494", "g-496", "g-498", "g-500", "g-503", "g-504", "g-506", "g-508", "g-509", "g-512",
"g-522", "g-529", "g-531", "g-534", "g-539", "g-541", "g-546", "g-551", "g-553", "g-554", "g-559", "g-561", "g-562", "g-565", "g-568", "g-569", "g-574",
"g-577", "g-578", "g-586", "g-588", "g-590", "g-594", "g-595", "g-596", "g-597", "g-599", "g-600", "g-603", "g-607", "g-615", "g-618", "g-619", "g-620",
"g-625", "g-628", "g-629", "g-632", "g-634", "g-635", "g-636", "g-638", "g-639", "g-641", "g-643", "g-644", "g-645", "g-646", "g-647", "g-648", "g-663",
"g-664", "g-665", "g-668", "g-669", "g-670", "g-671", "g-672", "g-673", "g-674", "g-677", "g-678", "g-680", "g-683", "g-689", "g-691", "g-693", "g-695",
"g-701", "g-702", "g-703", "g-704", "g-705", "g-706", "g-708", "g-711", "g-712", "g-720", "g-721", "g-723", "g-724", "g-726", "g-728", "g-731", "g-733",
"g-738", "g-739", "g-742", "g-743", "g-744", "g-745", "g-749", "g-750", "g-752", "g-760", "g-761", "g-764", "g-766", "g-768", "g-770", "g-771", "c-0",
"c-1", "c-2", "c-3", "c-4", "c-5", "c-6", "c-7", "c-8", "c-9", "c-10", "c-11", "c-12", "c-13", "c-14", "c-15", "c-16", "c-17", "c-18", "c-19", "c-20",
"c-21", "c-22", "c-23", "c-24", "c-25", "c-26", "c-27", "c-28", "c-29", "c-30", "c-31", "c-32", "c-33", "c-34", "c-35", "c-36", "c-37", "c-38", "c-39",
"c-40", "c-41", "c-42", "c-43", "c-44", "c-45", "c-46", "c-47", "c-48", "c-49", "c-50", "c-51", "c-52", "c-53", "c-54", "c-55", "c-56", "c-57", "c-58",
"c-59", "c-60", "c-61", "c-62", "c-63", "c-64", "c-65", "c-66", "c-67", "c-68", "c-69", "c-70", "c-71", "c-72", "c-73", "c-74", "c-75", "c-76", "c-77",
"c-78", "c-79", "c-80", "c-81", "c-82", "c-83", "c-84", "c-85", "c-86", "c-87", "c-88", "c-89", "c-90", "c-91", "c-92", "c-93", "c-94", "c-95", "c-96",
"c-97", "c-98", "c-99"]
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train, train_targets, test = mapping_and_filter(train, train_targets, test)
 train, test = fe_stats(train, test)
 train, test = c_squared(train, test)
 train, test = fe_pca(train, test, n_components_g = 70, n_components_c = 10, SEED = 123)
 train, test, features = scaling(train, test)
 test_pred_5l, oof_pred_5l = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS1, MODEL = '5l', PATH = '.. /input/moaaaaa/5l')
 test_pred_4l, oof_pred_4l = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS2, MODEL = '4l', PATH = '.. /input/moaaaaa/4l')
 test_pred_3l, oof_pred_3l = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS3, MODEL = '3l', PATH = '.. /input/moaaaaa/3l')
 test_pred_2l, oof_pred_2l = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS4, MODEL = '2l', PATH = '.. /input/moaaaaa/2l')
 test_pred_rs, oof_pred_rs = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS5, MODEL = 'rs', PATH = '.. /input/moaaaaa/rs')
 oof_pred = np.average([oof_pred_5l, oof_pred_4l, oof_pred_3l, oof_pred_2l], axis = 0)
 seed_log_loss = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our final out of folds log loss for our classic dnn blend is {seed_log_loss}')
 test_pred = np.average([test_pred_5l, test_pred_4l, test_pred_3l, test_pred_2l], axis = 0)
 oof_pred = np.average([oof_pred, oof_pred_rs], axis = 0)
 seed_log_loss = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our final out of folds log loss for our classic dnn + dnn resnet type model is {seed_log_loss}')
 test_pred = np.average([test_pred, test_pred_rs], axis = 0)
 print(type(oof_pred))
print(oof_pred.shape)
 train_targets.to_csv('sub3_label.csv', index = False)
 np.save("sub3.npy", oof_pred)
 np.save("sub3_label.npy", train_targets.values)
 sample_submission = submission(test_pred)
 sample_submission.head()
criterion = torch.nn.CrossEntropyLoss()
num_epochs = 5
train_loss, val_loss = [], []
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
 model.train()
running_loss = 0
tk0 = tqdm(train_loader, total=int(len(train_loader)))
 for im, labels in tk0:
inputs = im["image"].to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.long)
 optimizer.zero_grad()
outputs = model(inputs)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item()
tk0.set_postfix(loss=(loss.item()))
epoch_loss = running_loss /(len(train_loader)/batch_size)
 train_loss.append(epoch_loss)
 print('Training Loss: {:.8f}'.format(epoch_loss))
tk1 = tqdm(valid_loader, total=int(len(valid_loader)))
 model.eval()
running_loss = 0
y, preds = [], []
with torch.no_grad() :
for(im, labels)in tk1:
inputs = im["image"].to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.long)
 outputs = model(inputs)
 loss = criterion(outputs, labels)
 y.extend(labels.cpu().numpy().astype(int))
preds.extend(F.softmax(outputs, 1 ).cpu().numpy())
 running_loss += loss.item()
tk1.set_postfix(loss=(loss.item()))
epoch_loss = running_loss /(len(valid_loader)/batch_size)
 val_loss.append(epoch_loss)
 preds = np.array(preds)
 y = np.array(y)
 labels = preds.argmax(1)
 for class_label in np.unique(y):
idx = y == class_label
acc =(labels[idx] == y[idx] ).astype(np.float ).mean() *100
print('accuracy for class', class_names[class_label], 'is', acc)
 acc =(labels == y ).mean() *100
new_preds = np.zeros(( len(preds),))
temp = preds[labels != 0, 1:]
new_preds[labels != 0] = temp.sum(1)
 new_preds[labels == 0] = 1 - preds[labels == 0, 0]
y = np.array(y)
 y[y != 0] = 1
auc_score = alaska_weighted_auc(y, new_preds)
 print(
 f'Val Loss: {epoch_loss:.3}, Weighted AUC:{auc_score:.3}, Acc: {acc:.3}')
 torch.save(model.state_dict() ,
f"epoch_{epoch+dones}_val_loss_{epoch_loss:.3}_auc_{auc_score:.3}.pth")
<create_dataframe>
sub3 = sample_submission
class Alaska2TestDataset(Dataset):
def __init__(self, df, augmentations=None):
self.data = df
self.augment = augmentations
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
fn = self.data.loc[idx][0]
im = cv2.imread(fn)[:, :, ::-1]
if self.augment:
im = self.augment(image=im)
 return im
test_filenames = sorted(glob(f"{data_dir}/Test/*.jpg"))
test_df = pd.DataFrame({'ImageFileName': list(
 test_filenames)}, columns=['ImageFileName'])
 batch_size = 16
num_workers = 4
test_dataset = Alaska2TestDataset(test_df, augmentations=AUGMENTATIONS_TEST)
 test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=batch_size,
num_workers=num_workers,
shuffle=False,
drop_last=False )<find_best_params>
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
model.eval()
preds = []
tk0 = tqdm(test_loader)
 with torch.no_grad() :
for i, im in enumerate(tk0):
inputs = im["image"].to(device)
 im = inputs.flip(2)
 outputs = model(im)
 im = inputs.flip(3)
 outputs =(0.25*outputs + 0.25*model(im))
outputs =(outputs + 0.5*model(inputs))
preds.extend(F.softmax(outputs, 1 ).cpu().numpy())
 preds = np.array(preds)
 labels = preds.argmax(1)
 new_preds = np.zeros(( len(preds),))
new_preds[labels != 0] = preds[labels != 0, 1:].sum(1)
 new_preds[labels == 0] = 1 - preds[labels == 0, 0]
test_df['Id'] = test_df['ImageFileName'].apply(lambda x: x.split(os.sep)[-1])
 test_df['Label'] = new_preds
test_df = test_df.drop('ImageFileName', axis=1)
 test_df.to_csv('submission_eb0.csv', index=False)
 print(test_df.head() )<load_from_csv>
warnings.filterwarnings('ignore' )
train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )<prepare_x_and_y>
from sklearn.preprocessing import QuantileTransformer
X = train.drop('label',axis = 1)
 y = train.label<categorify>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
y = to_categorical(y )<split>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.25,random_state = 123 )<choose_model_class>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
model = Sequential()
model.add(Conv2D(32,kernel_size =(3,3),activation = 'relu',input_shape =(28,28,1)))
 model.add(Conv2D(64,kernel_size =(3,3),activation = 'relu'))
model.add(Flatten())
 model.add(Dense(10,activation = 'softmax')) 
<choose_model_class>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
model.compile(optimizer = 'adam',loss = 'categorical_crossentropy',metrics = ['accuracy'] )<train_model>
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
history = model.fit(X_train,y_train,validation_data =(X_test,y_test),epochs = 20 )<choose_model_class>
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))<choose_model_class>
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
model.compile(optimizer = 'Adam' , loss = "categorical_crossentropy", metrics=["accuracy"] )<train_model>
def fe_cluster(train, test, n_clusters_g = 35, n_clusters_c = 5, SEED = 123):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features ,test_features=fe_cluster(train_features,test_features )
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<init_hyperparams>
def fe_stats(train, test):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train, test
train_features,test_features=fe_stats(train_features,test_features )
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 batch_size = 256
history = model.fit_generator(datagen.flow(X_train,y_train, batch_size=batch_size),
epochs = 30, validation_data =(X_test,y_test),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )<predict_on_test>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
predictions = model.predict_classes(test, verbose=1 )<create_dataframe>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
prediction = pd.DataFrame({"ImageId":list(range(1,len(predictions)+1)) ,"Label":predictions} )<save_to_csv>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
prediction.to_csv('kaggle_submission.csv',index=False,header=True)
 prediction<choose_model_class>
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=5)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds
model = Sequential()
model.add(Conv2D(64,(3,3),activation='relu',input_shape=(28,28,1)))
 model.add(Conv2D(64,(3,3),activation='relu'))
model.add(MaxPooling2D(2,2))
model.add(BatchNormalization())
 model.add(Dropout(0.1))
model.add(Conv2D(128,(3,3),activation='relu'))
model.add(Conv2D(128,(3,3),activation='relu'))
model.add(MaxPooling2D(2,2))
model.add(BatchNormalization())
 model.add(Dropout(0.1))
model.add(Flatten())
 model.add(Dense(256,activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(128,activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(64,activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(10,activation='softmax')) 
<choose_model_class>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
model.compile(RMSprop(lr=0.001,rho=0.9),loss='categorical_crossentropy',metrics=['accuracy'] )<init_hyperparams>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
train_datagen = ImageDataGenerator(rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=False,
fill_mode='nearest')
 train_datagen.fit(X_train)
 train_generator = train_datagen.flow(X_train,y_train,batch_size=128)
<train_model>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
earlystop = EarlyStopping(monitor='val_loss',patience=2,verbose=1)
 learning_reduce = ReduceLROnPlateau(patience=2,monitor="val_acc",verbose=1,min_lr=0.00001,factor=0.5)
 callbacks = [learning_reduce]
history = model.fit_generator(train_generator,epochs=30,verbose=1,validation_data=(X_test,y_test),callbacks=callbacks )<define_variables>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.4)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.4)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
BATCH_SIZE = 64
VALID_BATCH_SIZE = 100
TEST_BATCH_SIZE = 100
EPOCHS = 5
NUM_CLASSES = 10
SEED = 42
EARLY_STOPPING = 25
OUTPUT_DIR = '/kaggle/working/'
MODEL_NAME = 'efficientnet-b0'<install_modules>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
!pip install efficientnet-pytorch<import_modules>
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim import lr_scheduler
from torch.autograd import Variable
from torch.utils.data import DataLoader, Dataset
from sklearn.metrics import accuracy_score
from PIL import Image, ImageOps, ImageEnhance
from efficientnet_pytorch import EfficientNet<load_from_csv>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 5
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 print('Shape of the training data: ', train.shape)
 print('Shape of the test data: ', test.shape )<split>
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
train_df, valid_df = train_test_split(train, test_size = 0.2, random_state=SEED,stratify=train['label'] )<prepare_x_and_y>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
n_pixels = len(train_df.columns)- 1
class MNIST_Dataset(Dataset):

 def __init__(self, df 
):
if len(df.columns)== n_pixels:
self.X = df.values.reshape(( -1,28,28)).astype(np.uint8)[:,:,:,None]
self.y = None
self.X3 = np.full(( self.X.shape[0], 3, 28, 28), 0.0)
 for i, s in enumerate(self.X):
self.X3[i] = np.moveaxis(cv2.cvtColor(s, cv2.COLOR_GRAY2RGB), -1, 0)
 else:
self.X = df.iloc[:,1:].values.reshape(( -1,28,28)).astype(np.uint8)[:,:,:,None]
self.y = torch.from_numpy(df.iloc[:,0].values)
 self.X3 = np.full(( self.X.shape[0], 3, 28, 28), 0.0)
 for i, s in enumerate(self.X):
self.X3[i] = np.moveaxis(cv2.cvtColor(s, cv2.COLOR_GRAY2RGB), -1, 0)
 def __len__(self):
return len(self.X3)
 def __getitem__(self, idx):
if self.y is not None:
return self.X3[idx] , self.y[idx]
else:
return self.X3[idx]<create_dataframe>
SEED = [0, 1, 2]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
train_dataset = MNIST_Dataset(train_df)
 valid_dataset = MNIST_Dataset(valid_df)
 test_dataset = MNIST_Dataset(test)
 train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=BATCH_SIZE,
shuffle=True)
 valid_loader = torch.utils.data.DataLoader(dataset=valid_dataset,
batch_size=VALID_BATCH_SIZE, shuffle=False)
 test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=TEST_BATCH_SIZE, shuffle=False )<choose_model_class>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
print(type(valid_results))
print(valid_results.shape)
 valid_results.to_csv('sub4.csv', index=False)
 train_targets_scored.to_csv('sub4_label.csv', index=False)
 score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
def get_model(model_name='efficientnet-b0'):
model = EfficientNet.from_pretrained(model_name)
 del model._fc
model._fc = nn.Linear(1280, NUM_CLASSES)
 return model<set_options>
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False)
 sub.shape
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed )<choose_model_class>
sub4 = sub
set_seed(SEED)
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 output_dir = OUTPUT_DIR
model = get_model(MODEL_NAME)
 model = model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=0.001)
 scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)
 loss_func = nn.CrossEntropyLoss()
if torch.cuda.is_available() :
model = model.cuda()
loss_func = loss_func.cuda()
best_val_accuracy = 0
min_val_loss = np.inf
best_epoch = 0
batches = 0
epochs_no_improve = 0
n_epochs_stop = EARLY_STOPPING
for epoch in range(EPOCHS):
running_loss = 0.0
targets = torch.empty(size=(BATCH_SIZE,)).to(device)
 outputs = torch.empty(size=(BATCH_SIZE,)).to(device)
 model.train()
for batch_idx,(data, target)in enumerate(train_loader):
batches += 1
data, target = Variable(data), Variable(target)
 if torch.cuda.is_available() :
data = data.type(torch.FloatTensor ).cuda()
target = target.cuda()
targets = torch.cat(( targets, target), 0)
 optimizer.zero_grad()
output = model(data)
 loss = loss_func(output, target)
 output = torch.argmax(torch.softmax(output, dim=1), dim=1)
 outputs = torch.cat(( outputs, output), 0)
 running_loss += loss.item()
loss.backward()
optimizer.step()
scheduler.step()
print('train/loss on EPOCH {}: {}'.format(epoch, running_loss/batches))
train_acc = accuracy_score(targets.cpu().detach().numpy().astype(int),
outputs.cpu().detach().numpy().astype(int))
print('train/accuracy: {} for epoch {}'.format(train_acc, epoch))
model.eval()
running_loss = 0.0
batches = 0
targets = torch.empty(size=(BATCH_SIZE,)).to(device)
 outputs = torch.empty(size=(BATCH_SIZE,)).to(device)
 for batch_idx,(data, target)in enumerate(valid_loader):
batches += 1
data, target = Variable(data, volatile=True), Variable(target)
 if torch.cuda.is_available() :
data = data.type(torch.FloatTensor ).cuda()
target = target.cuda()
with torch.no_grad() :
targets = torch.cat(( targets, target), 0)
 output = model(data)
 loss = loss_func(output, target)
 output = torch.argmax(torch.softmax(output, dim=1), dim=1)
 outputs = torch.cat(( outputs, output), 0)
 running_loss += loss.item()
val_loss = running_loss/batches
print('val/loss: {}'.format(val_loss))
val_acc = accuracy_score(targets.cpu().detach().numpy().astype(int),
outputs.cpu().detach().numpy().astype(int))
print('val/accuracy: {} for epoch {}'.format(val_acc, epoch))
if val_acc > best_val_accuracy:
best_val_accuracy = val_acc
min_val_loss = val_loss
print('Best val/acc: {} for epoch {}, saving model---->'.format(val_acc, epoch))
torch.save(model.state_dict() , "{}/snapshot_epoch_{}.pth".format(output_dir, epoch))
best_epoch = epoch
epochs_no_improve = 0
else:
epochs_no_improve += 1
if epochs_no_improve == n_epochs_stop:
print('Early stopping!')
 break<predict_on_test>
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv' )
def prediction(model, data_loader):
model.eval()
test_pred = torch.LongTensor()
for i, data in enumerate(data_loader):
data = Variable(data, volatile=True)
 if torch.cuda.is_available() :
data = data.type(torch.FloatTensor ).cuda()
output = model(data)
 pred = output.cpu().data.max(1, keepdim=True)[1]
test_pred = torch.cat(( test_pred, pred), dim=0)
 return test_pred<predict_on_test>
test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_ids = list(test['sig_id'].values)
 private_ids = list(set(test_ids)-set(public_ids))
len(private_ids )
model.load_state_dict(torch.load("snapshot_epoch_{}.pth".format(best_epoch)))
 test_pred = prediction(model, test_loader)
 submission = pd.DataFrame(np.c_[np.arange(1, len(test_dataset)+1)[:,None], test_pred.numpy() ],
columns=['ImageId', 'Label'])
<save_to_csv>
predictions = [sub4,sub3,sub2,sub1]
submission.to_csv('submission.csv', index=False )<save_to_csv>
y_pred = pd.DataFrame()
y_pred['sig_id'] = predictions[0]['sig_id']
for column in target_columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column])
 y_pred[column] = np.mean(column_data, axis=0)
 y_pred.shape
<import_modules><EOS> 
submission = pd.DataFrame(index = public_ids + private_ids, columns=target_columns)
 submission.index.name = 'sig_id'
submission[:] = 0
submission.loc[y_pred.sig_id,:] = y_pred[target_columns].values
submission.loc[test[test.cp_type == 'ctl_vehicle'].sig_id] = 0
submission.to_csv('submission.csv', index=True )