Kaggle competition: RSNA STR Pulmonary Embolism Detection
<SOS> metric: weightedmeancolumnwiselogloss Kaggle data source: rsna-str-pulmonary-embolism-detection<concatenate>
starttime = time.time()
sub5=pd.concat([test_id, ansamble3],axis=1 )<save_to_csv>
!cp.. /input/gdcm-conda-install/gdcm.tar.
 !tar -xvzf gdcm.tar
!conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2
sub5.to_csv('TMDB_ansamble3.csv',index=False )<import_modules>
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 100 )
py.init_notebook_mode(connected=True)
 plt.style.use('ggplot')
 print(os.listdir(".. /input")) 
<load_from_csv>
col_index = 'SOPInstanceUID'
col_groupby = 'StudyInstanceUID'
df_test_path = ".. /input/rsna-str-pulmonary-embolism-detection/test.csv"
df_train_path = ".. /input/rsna-str-pulmonary-embolism-detection/train.csv"
df_sub_path = ".. /input/rsna-str-pulmonary-embolism-detection/sample_submission.csv"
test_image_path = ".. /input/rsna-str-pulmonary-embolism-detection/test/"
train_image_path = ".. /input/rsna-str-pulmonary-embolism-detection/train/"
LEN_PUBLIC = 146853
DEBUG = False
num_cpu = multiprocessing.cpu_count()
num_features1_1 = 1280
num_features1_2 = 1408
weight_dir1_1 = ".. /input/rsna2020-1/201022_6_CNN_b0_1loss_512_fp16_bs80_lr1e3(2)/201022_6_CNN_b0_1loss_512_fp16_bs80_lr1e3"
weight_dir2_1 = ".. /input/rsna2020-1/201025_3_2ndNNs_features_pool_flip_new/201025_3_2ndNNs_features_pool_flip_new"
weight_dir1_2 = ".. /input/rsna2020-1/201024_5_b2_1loss_512_fp16_bs80_lr1e3/201024_5_b2_1loss_512_fp16_bs80_lr1e3"
weight_dir2_2 = ".. /input/rsna2020-1/201026_3_2ndNNs_features_pool_flip_b0b2/201026_3_2ndNNs_features_pool_flip_b0b2"
do_full = False
BATCH_SIZE = 64
NUM_FOLD = 5
TIME_LIMIT = 60*60*7
LIMIT_OVER = False
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 train_df.head()<feature_engineering>
col_targets = [
'negative_exam_for_pe',
'indeterminate',
'chronic_pe',
'acute_and_chronic_pe',
'central_pe',
'leftsided_pe',
'rightsided_pe',
'rv_lv_ratio_gte_1',
'rv_lv_ratio_lt_1',
'pe_present_on_image',
]
def fix_date(x):

 year = x.split('/')[2]
if int(year)<= 19:
return x[:-2] + '20' + year
else:
return x[:-2] + '19' + year
train_df['release_date'] = pd.to_datetime(train_df['release_date'].apply(lambda x: fix_date(x)))
 test_df.loc[test_df['release_date'].isnull() == True, 'release_date'] = '01/01/98'
test_df['release_date'] = pd.to_datetime(test_df['release_date'].apply(lambda x: fix_date(x)))
 train_df['release_date'].head()<data_type_conversions>
if os.path.exists('.. /input/rsna-str-pulmonary-embolism-detection/train')and not do_full:
df_test_full=pd.read_csv(df_test_path)
 df_test_full_study = df_test_full[df_test_full[col_groupby].duplicated() ==False]
df_test=pd.read_csv(df_test_path ).head(2000)
 else:
df_test_full=pd.read_csv(df_test_path)
 df_test_full_study = df_test_full[df_test_full[col_groupby].duplicated() ==False]
df_test=pd.read_csv(df_test_path)
 print(df_test.shape)
 df_test.head()
def process_date(df):
date_parts = ["year", "weekday", "month", 'weekofyear', 'day', 'quarter']
for part in date_parts:
part_col = 'release_date' + "_" + part
df[part_col] = getattr(df['release_date'].dt, part ).astype(int)
 return df
train_df = process_date(train_df)
 test_df = process_date(test_df )<feature_engineering>
df_test = df_test.sort_values([col_groupby, col_index] ).reset_index(drop=True)
 df_test.head()
dict_columns = ['belongs_to_collection', 'genres', 'production_companies', 'production_countries',
'spoken_languages', 'Keywords', 'cast', 'crew']
def convert_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: ast.literal_eval(x)if not pd.isnull(x)else np.NaN)
 return df<categorify>
df_test['dicom_path'] = test_image_path + "/" +\
df_test[col_groupby].values + "/" + \
df_test['SeriesInstanceUID'].values + "/" + \
df_test[col_index].values + ".dcm"
print(df_test['dicom_path'][0])
 df_test.head()
train_df = convert_dict(train_df)
 test_df = convert_dict(test_df )<count_missing_values>
df_test_study = df_test[df_test[col_groupby].duplicated() ==False]
df_test_study['start_index'] = df_test_study.index.values
df_tmp = df_test.groupby(col_groupby)[col_index].agg(len ).reset_index()
df_tmp.columns = [col_groupby, 'num_images']
df_test_study = pd.merge(df_test_study, df_tmp, on=col_groupby)
 print(df_test_study.shape)
 df_test_study.head()
train_df['belongs_to_collection'].isnull().sum()<feature_engineering>
class DicomDataset(Dataset):
def __init__(self, X_study, X_image, transform=None, meta=False, verbose=False):
self.X_study = X_study
self.X_image = X_image
self.transform = transform
self.verbose = verbose
def __getitem__(self, index):
study = self.X_study[col_groupby][index]
start_index = self.X_study['start_index'][index]
end_index = self.X_study['start_index'][index] + self.X_study['num_images'][index]
df_study = self.X_image.iloc[start_index:end_index].reset_index(drop=True)
 images_study = []
z_pos = []
for i in range(len(df_study)) :
tmp_path = df_study['dicom_path'][i]
try:
tmp_dcm = pydicom.dcmread(tmp_path)
 tmp_npy = np.asarray(tmp_dcm.pixel_array)
 images_study.append(tmp_npy)
 if i==0:
RescaleSlope = tmp_dcm['RescaleSlope'].value
RescaleIntercept = tmp_dcm['RescaleIntercept'].value
PatientPosition = tmp_dcm['PatientPosition'].value
z_pos.append(tmp_dcm['ImagePositionPatient'].value[-1])
 except:
print("loading error!!!, study: {}, index: {}".format(study, i))
tmp_npy = np.zeros([512, 512], np.int16)
 images_study.append(tmp_npy)
 if i==0:
RescaleSlope = 1
RescaleIntercept = -1024
PatientPosition = 'HFS'
z_pos.append(-10000-i)
 images_study = np.array(images_study)
 z_pos = np.array(z_pos)
 images_study = images_study[np.argsort(z_pos)]
df_study['z_pos'] = z_pos
df_study = df_study.sort_values('z_pos' ).reset_index(drop=True)
 df_study['series_index'] = np.arange(len(df_study))
if self.verbose: print(images_study.shape)
 if self.verbose: print(z_pos)
 if self.verbose: print(RescaleIntercept, RescaleSlope, PatientPosition)
 images_study_processed =(images_study.astype(np.float32)* RescaleSlope + RescaleIntercept)/1000
if PatientPosition=='FFP':
images_study_processed = images_study_processed[:, ::-1, ::-1]
images_study_processed = images_study_processed.reshape([-1, 1, 512, 512] ).astype(np.float16)
 return images_study_processed, df_study
def __len__(self):
return len(self.X_study )
train_df['has_collection'] = train_df['belongs_to_collection'].apply(lambda x: 1 if not pd.isnull(x)else 0)
 test_df['has_collection'] = test_df['belongs_to_collection'].apply(lambda x: 1 if not pd.isnull(x)else 0)
 train_df['has_collection'][:5]<feature_engineering>
def my_collate(batch):
return torch.Tensor(batch[0][0]), batch[0][1]
train_df['collection_name'] = train_df['belongs_to_collection'].apply(lambda x: x[0]['name'] if not pd.isnull(x)else 0)
 test_df['collection_name'] = test_df['belongs_to_collection'].apply(lambda x: x[0]['name'] if not pd.isnull(x)else 0)
 train_df['collection_name'][:5]<drop_column>
class nnWindow(nn.Module):
def __init__(self):
super(nnWindow, self ).__init__()
wso = np.array(((40,80),(80,200),(40,400)))/1000
conv_ = nn.Conv2d(1,3, kernel_size=(1, 1))
conv_.weight.data.copy_(torch.tensor([[[[1./wso[0][1]]]],[[[1./wso[1][1]]]],[[[1./wso[2][1]]]]]))
conv_.bias.data.copy_(torch.tensor([0.5 - wso[0][0]/wso[0][1],
0.5 - wso[1][0]/wso[1][1],
0.5 -wso[2][0]/wso[2][1]]))
self.window = nn.Sequential(
 conv_,
nn.Sigmoid() ,
nn.InstanceNorm2d(3)
)
 def forward(self, input1):
return self.window(input1)
 class MyEffNet_b0(nn.Module):
def __init__(self, num_classes=10, base_model='tf_efficientnet_b0_ns'):
super(MyEffNet_b0, self ).__init__()
self.num_classes = num_classes
self.mode = 'train'
self.window = nnWindow()
self.base_model = timm.create_model(base_model, pretrained=False, num_classes=10 ).to(device, non_blocking=True)
 self.avgpool = nn.AdaptiveAvgPool2d(1)
 self.last_linear = nn.Linear(self.base_model.num_features, num_classes)
 def forward(self, input1):
bs, ch, h, w = input1.size()
x = self.window(input1)
 x = self.base_model.forward_features(x)
 feature = self.avgpool(x ).view(bs, -1)
 y = self.last_linear(feature)
 return y
def feature(self, input1):
bs, ch, h, w = input1.size()
x = self.window(input1)
 x = self.base_model.forward_features(x)
 feature = self.avgpool(x ).view(bs, -1)
 y = self.last_linear(feature)
 return y, feature
train_df = train_df.drop("belongs_to_collection", axis =1)
 test_df = test_df.drop("belongs_to_collection", axis =1 )<feature_engineering>
class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool1d(1)
 self.fc1 = nn.Conv1d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv1d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class CNN1D(nn.Module):
def __init__(self, num_classes=400, input_ch=1, verbose=False):
super(CNN1D, self ).__init__()
pool = 4
drop = 0.1
self.verbose = verbose
self.layer1 = nn.Sequential(
 nn.Conv1d(input_ch//pool, 64, kernel_size=7, stride=1, padding=3, bias=False),
nn.BatchNorm1d(64),
nn.ReLU(inplace=True),
SEModule(64, 16), 
)
 self.fpool = nn.MaxPool1d(kernel_size=pool, stride=pool, padding=0)
 self.maxpool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1)
 self.layer2 = nn.Sequential(
 nn.Conv1d(64, 128, kernel_size=3, stride=1, padding=1, bias=False),
nn.BatchNorm1d(128),
nn.ReLU(inplace=True),
SEModule(128, 16), 
)
 self.layer3 = nn.Sequential(
 nn.Conv1d(128, 256, kernel_size=3, stride=1, padding=1, bias=False),
nn.BatchNorm1d(256),
nn.ReLU(inplace=True),
SEModule(256, 16), 
)
 self.layer4 = nn.Sequential(
 nn.Conv1d(256, 512, kernel_size=3, stride=1, padding=1, bias=False),
nn.BatchNorm1d(512),
nn.ReLU(inplace=True),
SEModule(512, 16), 
)
 self.avgpool = nn.AdaptiveAvgPool1d(1)
 self.fc2 = nn.Conv1d(
 input_ch//pool+64+128+256+512,
2, kernel_size=1)
 self.fc = nn.Sequential(
 nn.Linear(512, 512),
nn.ReLU(inplace=True),
nn.Dropout(0.5),
nn.Linear(512, 512),
nn.ReLU(inplace=True),
nn.Dropout(0.5),
nn.Linear(512, 9), 
)
 def forward(self, x_input):
bs, ch, d = x_input.size()
x0 = torch.transpose(x_input, 1, 2)
 x0 = self.fpool(x0)
 x0 = torch.transpose(x0, 1, 2)
 x1 = self.layer1(x0)
 x1 = self.maxpool(x1)
 x2 = self.layer2(x1)
 x2 = self.maxpool(x2)
 x3 = self.layer3(x2)
 x3 = self.maxpool(x3)
 x4 = self.layer4(x3)
 x5 = torch.cat([
x0,
F.adaptive_avg_pool1d(x1, d),
F.adaptive_avg_pool1d(x2, d),
F.adaptive_avg_pool1d(x3, d),
F.adaptive_avg_pool1d(x4, d),
], axis=1)
 y2 = self.fc2(x5)
 b, ch, d = x_input.size()
y = self.avgpool(x4)
 y = y.view(b, -1)
 y = self.fc(y)
 return y, y2
def dictToInd(var, num):
train_df[var] = train_df[var].fillna(0)
 test_df[var] = test_df[var].fillna(0)
 train_df['num_' + var] = train_df[var].apply(lambda x: len(x)if x!= 0 else 0)
 test_df['num_' + var] = test_df[var].apply(lambda x: len(x)if x!= 0 else 0)
 train_df['all_'+var] = train_df[var].apply(lambda x: [i['name'] for i in x] if x !=0 else 0)
 test_df['all_'+var] = test_df[var].apply(lambda x: [i['name'] for i in x] if x !=0 else 0)
 d = dict()
for i, e in enumerate(train_df['all_' + var]):
if e != 0:
for k in e:
if k in d:
d[k] += 1
else:
d[k] = 1
top_d = dict(sorted(d.items() , key = operator.itemgetter(1), reverse = True)[:num])
 for i in top_d:
train_df[var +'_' + i] = train_df['all_' + var].apply(lambda x: 1 if x!=0 and i in x else 0)
 test_df[var +'_' + i] = test_df['all_' + var].apply(lambda x: 1 if x!=0 and i in x else 0 )<count_values>
dataset_test = DicomDataset(df_test_study, df_test)
 test_loader = DataLoader(
 dataset_test,
batch_size=1,
shuffle=False,
num_workers=num_cpu,
pin_memory=True,
collate_fn=my_collate
)
train_df['num_genres'].value_counts()<define_variables>
lastfunc = nn.Sigmoid().to(device, non_blocking=True )
genres = []
for i, e in enumerate(train_df['all_genres']):
if e != 0:
for genre in e:
if genre not in genres:
genres.append(genre)
 print(genres )<count_values>
def batch_padding(batch):
bs, ch, d = batch.shape
d_new = int(np.ceil(d/64)*64)
 batch_new = torch.from_numpy(np.zeros([bs, ch, d_new], np.float32)).to(device, non_blocking=True)
 batch_new[:, :, :d] = batch
return batch_new
train_df['production_companies'].fillna(0 ).apply(lambda x: len(x)if x != 0 else 0 ).value_counts()<count_values>
NUM_FOLDS = 5
df_pred_image = []
df_pred_study = []
starttime = time.time()
verbose = False
model_b0s = []
model_b2s = []
model_1dcnns = []
model_1dcnn_b2s = []
for fold in range(NUM_FOLDS):
model_b0s.append(MyEffNet_b0().to(device, non_blocking=True))
model_b2s.append(MyEffNet_b0(base_model='tf_efficientnet_b2_ns' ).to(device, non_blocking=True))
model_1dcnns.append(CNN1D(input_ch=num_features1_1 ).to(device, non_blocking=True))
model_1dcnn_b2s.append(CNN1D(input_ch=num_features1_2 ).to(device, non_blocking=True))
model_b0s[fold].load_state_dict(torch.load("{}/weight_epoch_16_fold{}.pth".format(weight_dir1_1, fold+1)))
 model_b2s[fold].load_state_dict(torch.load("{}/weight_epoch_16_fold{}.pth".format(weight_dir1_2, fold+1)))
 model_1dcnns[fold].load_state_dict(torch.load("{}/cnn_weight_best_fold{}.pth".format(weight_dir2_1, fold+1)))
 model_1dcnn_b2s[fold].load_state_dict(torch.load("{}/cnn_weight_best_fold{}.pth".format(weight_dir2_2, fold+1)))
 model_b0s[fold].eval()
model_b2s[fold].eval()
model_1dcnns[fold].eval()
model_1dcnn_b2s[fold].eval()
for study_index,(images, df_study)in enumerate(test_loader):
if time.time() -starttime>TIME_LIMIT and LIMIT_OVER==False:
print("index: {}, {:.1f}, time limit over!!!".format(study_index, time.time() -starttime))
LIMIT_OVER = True
if study_index>=100: break
if verbose: print("load index {} done".format(study_index), time.time() -starttime)
 if(study_index+1)%10==0:
print("{}/{}, sec: {:.1f}".format(study_index+1, len(df_test_study), time.time() -starttime))
num_batches = int(np.ceil(images.shape[0]/BATCH_SIZE))
num_images = len(df_study)
 df_study_image = df_study[[col_groupby, col_index, 'SeriesInstanceUID']]
df_study_study = df_study[[col_groupby]].iloc[:1]
for fold in range(NUM_FOLD):
model_b0 = model_b0s[fold]
model_b2 = model_b2s[fold]
model_1dcnn = model_1dcnns[fold]
model_1dcnn_b2 = model_1dcnn_b2s[fold]
features = []
features_b2 = []
for batch_index in range(num_batches):
with torch.no_grad() :
with torch.cuda.amp.autocast() :
batch = images[batch_index*BATCH_SIZE:(batch_index+1)*BATCH_SIZE].to(device, non_blocking=True)
 _, feature = model_b0.feature(batch)
 if LIMIT_OVER!=True:
_, feature_b2 = model_b2.feature(batch)
 features.append(feature)
 features_b2.append(feature_b2)
 features = torch.cat(features, axis=0)
 features = torch.transpose(features, 0,1 ).reshape([1, num_features1_1, -1])
 features = batch_padding(features)
 if LIMIT_OVER!=True:
features_b2 = torch.cat(features_b2, axis=0)
 features_b2 = torch.transpose(features_b2, 0,1 ).reshape([1, num_features1_2, -1])
 features_b2 = batch_padding(features_b2)
 with torch.no_grad() :
output1, output2 = model_1dcnn(features)
 output2 = output2[:,-1:]
output1 = lastfunc(output1)
 output2 = lastfunc(output2)[:,:,:num_images]
if LIMIT_OVER!=True:
output1_b2, output2_b2 = model_1dcnn_b2(features_b2)
 output2_b2 = output2_b2[:,-1:]
output1_b2 = lastfunc(output1_b2)
 output2_b2 = lastfunc(output2_b2)[:,:,:num_images]
for i, col in enumerate(col_targets[:-1]):
df_study_study["{}_pred_fold{}".format(col, fold+1)] = output1[0, i].data.cpu().numpy()
if LIMIT_OVER!=True:
df_study_study["{}_pred_fold{}_b2".format(col, fold+1)] = output1_b2[0, i].data.cpu().numpy()
else:
df_study_study["{}_pred_fold{}_b2".format(col, fold+1)] = output1[0, i].data.cpu().numpy()
df_study_image["{}_pred_fold{}".format(col_targets[-1], fold+1)] = output2[0, 0].data.cpu().numpy()
if LIMIT_OVER!=True:
df_study_image["{}_pred_fold{}_b2".format(col_targets[-1], fold+1)] = output2_b2[0, 0].data.cpu().numpy()
else:
df_study_image["{}_pred_fold{}_b2".format(col_targets[-1], fold+1)] = output2[0, 0].data.cpu().numpy()
df_pred_study.append(df_study_study)
 df_pred_image.append(df_study_image)
train_df['production_countries'].fillna(0 ).apply(lambda x: len(x)if x != 0 else 0 ).value_counts()<count_values>
df_pred_image = pd.concat(df_pred_image ).reset_index(drop=True)
 df_pred_study = pd.concat(df_pred_study ).reset_index(drop=True)
 print(df_pred_image.shape, df_pred_study.shape)
 df_pred_image.head()
train_df['spoken_languages'].fillna(0 ).apply(lambda x: len(x)if x != 0 else 0 ).value_counts()<count_values>
for col in col_targets[:-1]:
cols_tmp = []
for fold in range(NUM_FOLD):
cols_tmp.append("{}_pred_fold{}".format(col, fold+1))
cols_tmp.append("{}_pred_fold{}_b2".format(col, fold+1))
df_pred_study[col] = df_pred_study[cols_tmp].values.mean(axis=1)
 df_pred_study.head(30 )
train_df['Keywords'].fillna(0 ).apply(lambda x: len(x)if x !=0 else 0 ).value_counts()<define_variables>
cols_tmp = []
for fold in range(NUM_FOLD):
cols_tmp.append("{}_pred_fold{}".format(col_targets[-1], fold+1))
cols_tmp.append("{}_pred_fold{}_b2".format(col_targets[-1], fold+1))
df_pred_image[col_targets[-1]] = df_pred_image[cols_tmp].values.mean(axis=1)
 df_pred_image.head()
list_of_keywords = list(train_df['Keywords'].fillna(0 ).apply(lambda x: [i['name'] for i in x] if x != 0 else [] ).values)
 list_of_keywords[:19]<count_values>
def solve_conflict(df_pred_s, df_pred, TH_NEGATIVE=0.5, TH_INDETERMINATE = 0.5, verbose=True):
index_indeterminate = df_pred_s['indeterminate']>TH_INDETERMINATE
index_negative =(index_indeterminate==False)&(df_pred_s['negative_exam_for_pe']>TH_NEGATIVE)
 index_positive =(index_indeterminate==False)&(index_negative==False)
 index_negative_and_negative_lte_05 = index_negative &(df_pred_s['negative_exam_for_pe']<=0.5)
 df_pred_s['negative_exam_for_pe'][index_negative_and_negative_lte_05] = 0.5001
index_indeterminate_and_indeterminate_lte_05 = index_indeterminate &(df_pred_s['indeterminate']<=0.5)
 df_pred_s['indeterminate'][index_indeterminate_and_indeterminate_lte_05] = 0.5001
index_indeterminate_and_negative_gt_05 = index_indeterminate &(df_pred_s['negative_exam_for_pe']>0.5)
 df_pred_s['negative_exam_for_pe'][index_indeterminate_and_negative_gt_05] = 0.5
index_negative_and_indeterminate_gt_05 = index_negative &(df_pred_s['indeterminate']>0.5)
 df_pred_s['indeterminate'][index_negative_and_indeterminate_gt_05] = 0.5
index_positive_and_negative_gt_05 = index_positive &(df_pred_s['negative_exam_for_pe']>0.5)
 df_pred_s['negative_exam_for_pe'][index_positive_and_negative_gt_05] = 0.5
index_positive_and_indeterminate_gt_05 = index_positive &(df_pred_s['indeterminate']>0.5)
 df_pred_s['indeterminate'][index_positive_and_indeterminate_gt_05] = 0.5
index_negative_and_rv_lv_ratio_lt_1_gt_05 =(index_positive==False)&(df_pred_s['rv_lv_ratio_lt_1']>0.5)
 df_pred_s['rv_lv_ratio_lt_1'][index_negative_and_rv_lv_ratio_lt_1_gt_05] = 0.5
index_negative_and_rv_lv_ratio_gte_1_gt_05 =(index_positive==False)&(df_pred_s['rv_lv_ratio_gte_1']>0.5)
 df_pred_s['rv_lv_ratio_gte_1'][index_negative_and_rv_lv_ratio_gte_1_gt_05] = 0.5
index_negative_and_central_pe_gt_05 =(index_positive==False)&(df_pred_s['central_pe']>0.5)
 index_negative_and_rightsided_pe_gt_05 =(index_positive==False)&(df_pred_s['rightsided_pe']>0.5)
 df_pred_s['rightsided_pe'][index_negative_and_rightsided_pe_gt_05] = 0.5
index_negative_and_leftsided_pe_gt_05 =(index_positive==False)&(df_pred_s['leftsided_pe']>0.5)
 df_pred_s['leftsided_pe'][index_negative_and_leftsided_pe_gt_05] = 0.5
index_negative_and_chronic_pe_gt_05 =(index_positive==False)&(df_pred_s['chronic_pe']>0.5)
 df_pred_s['chronic_pe'][index_negative_and_chronic_pe_gt_05] = 0.5
index_negative_and_acute_and_chronic_pe_gt_05 =(index_positive==False)&(df_pred_s['acute_and_chronic_pe']>0.5)
 df_pred_s['acute_and_chronic_pe'][index_negative_and_acute_and_chronic_pe_gt_05] = 0.5
index_positive_and_rv_gte_lv = index_positive &(df_pred_s['rv_lv_ratio_lt_1']<=df_pred_s['rv_lv_ratio_gte_1'])
 index_positive_and_rv_lt_lv = index_positive &(df_pred_s['rv_lv_ratio_lt_1']>df_pred_s['rv_lv_ratio_gte_1'])
 index_positive_and_rv_gte_lv_and_rv_lv_ratio_gte_1_lte_05 =\ 
(index_positive_and_rv_gte_lv)&(df_pred_s['rv_lv_ratio_gte_1']<=0.5)
 df_pred_s['rv_lv_ratio_gte_1'][index_positive_and_rv_gte_lv_and_rv_lv_ratio_gte_1_lte_05] = 0.5001
index_positive_and_rv_gte_lv_and_rv_lv_ratio_lt_1_gt_05 =\ 
(index_positive_and_rv_gte_lv)&(df_pred_s['rv_lv_ratio_lt_1']>0.5)
 df_pred_s['rv_lv_ratio_lt_1'][index_positive_and_rv_gte_lv_and_rv_lv_ratio_lt_1_gt_05] = 0.5
index_positive_and_rv_lt_lv_and_rv_lv_ratio_lt_1_lte_05 =\ 
(index_positive_and_rv_lt_lv)&(df_pred_s['rv_lv_ratio_lt_1']<=0.5)
 df_pred_s['rv_lv_ratio_lt_1'][index_positive_and_rv_lt_lv_and_rv_lv_ratio_lt_1_lte_05] = 0.5001
index_positive_and_rv_lt_lv_and_rv_lv_ratio_gte_1_gt_05 =\ 
(index_positive_and_rv_lt_lv)&(df_pred_s['rv_lv_ratio_gte_1']>0.5)
 df_pred_s['rv_lv_ratio_gte_1'][index_positive_and_rv_lt_lv_and_rv_lv_ratio_gte_1_gt_05] = 0.5
index_positive_and_central_is_greatest = index_positive &(df_pred_s['central_pe']>=df_pred_s['rightsided_pe'])&(df_pred_s['central_pe']>=df_pred_s['leftsided_pe'])
 index_positive_and_right_is_greatest = index_positive &(index_positive_and_central_is_greatest==False)&(df_pred_s['rightsided_pe']>=df_pred_s['leftsided_pe'])
 index_positive_and_left_is_greatest = index_positive &(index_positive_and_central_is_greatest==False)&(index_positive_and_right_is_greatest==False)
 index_positive_and_central_is_greatest_and_central_pe_lte_05 =(index_positive_and_central_is_greatest)&(df_pred_s['central_pe']<=0.5)
 df_pred_s['central_pe'][index_positive_and_central_is_greatest_and_central_pe_lte_05] = 0.5001
index_positive_and_right_is_greatest_and_rightsided_pe_lte_05 =(index_positive_and_right_is_greatest)&(df_pred_s['rightsided_pe']<=0.5)
 df_pred_s['rightsided_pe'][index_positive_and_right_is_greatest_and_rightsided_pe_lte_05] = 0.5001
index_positive_and_left_is_greatest_and_leftsided_pe_lte_05 =(index_positive_and_left_is_greatest)&(df_pred_s['leftsided_pe']<=0.5)
 df_pred_s['leftsided_pe'][index_positive_and_left_is_greatest_and_leftsided_pe_lte_05] = 0.5001
index_double_positive = index_positive &(df_pred_s['chronic_pe']>0.5)&(df_pred_s['acute_and_chronic_pe']>0.5)
 index_double_positive_and_chronic_lte_acute_and_chronic = index_double_positive &(df_pred_s['chronic_pe']<=df_pred_s['acute_and_chronic_pe'])
 df_pred_s['chronic_pe'][index_double_positive_and_chronic_lte_acute_and_chronic] = 0.5
index_double_positive_and_chronic_gt_acute_and_chronic = index_double_positive &(df_pred_s['chronic_pe']>df_pred_s['acute_and_chronic_pe'])
 df_pred_s['acute_and_chronic_pe'][index_double_positive_and_chronic_gt_acute_and_chronic] = 0.5
df_pred_s['positive'] = 0
df_pred_s['positive'][index_positive] = 1
df_pred2 = pd.merge(df_pred, df_pred_s[[col_groupby, 'positive']], on=col_groupby, how='left')
 df_agg = df_pred.groupby(col_groupby)['pe_present_on_image'].agg('max' ).reset_index()
df_agg.columns = [col_groupby, 'pe_present_on_image_pred_max']
df_pred2 = pd.merge(df_pred2, df_agg, on=col_groupby, how='left')
 df_pred2['peak'] = df_pred2['pe_present_on_image']==df_pred2['pe_present_on_image_pred_max']
index_positive_i = df_pred2['positive']==1
index_negative_and_pe_present_on_image_gt_05_i =(index_positive_i==False)&(df_pred2['pe_present_on_image']>0.5)
 df_pred['pe_present_on_image'][index_negative_and_pe_present_on_image_gt_05_i] = 0.5
index_positive_and_peak_and_pe_present_on_image_lte_05_i = index_positive_i &(df_pred2['peak'])&(df_pred2['pe_present_on_image']<=0.5)
 df_pred['pe_present_on_image'][index_positive_and_peak_and_pe_present_on_image_lte_05_i] = 0.5001
if verbose:
print("num study", len(df_pred_s))
print("num image", len(df_pred))
print("split to 3 classes")
 print(" num predicted_as_negative:", index_negative.sum())
 print(" num predicted_as_indeterminate:", index_indeterminate.sum())
 print(" num predicted_as_positive:", index_positive.sum())
 print("process 3 class conflict")
 print(" num predicted_as_negative and negative<=0.5:", index_negative_and_negative_lte_05.sum())
 print(" num predicted_as_indeterminate and indeterminate<=0.5:", index_indeterminate_and_indeterminate_lte_05.sum())
 print(" num predicted_as_indeterminate and negative_exam_for_pe>0.5:", index_indeterminate_and_negative_gt_05.sum())
 print(" num predicted_as_negative and indeterminate>0.5:", index_negative_and_indeterminate_gt_05.sum())
 print(" num predicted_as_positive and negative_exam_for_pe>0.5:", index_positive_and_negative_gt_05.sum())
 print(" num predicted_as_positive and indeterminate>0.5:", index_positive_and_indeterminate_gt_05.sum())
 print("process negative case")
 print(" num predicted_as_not_positive and rv_lv_ratio_lt_1>0.5:", index_negative_and_rv_lv_ratio_lt_1_gt_05.sum())
 print(" num predicted_as_not_positive and rv_lv_ratio_gte_1>0.5:", index_negative_and_rv_lv_ratio_gte_1_gt_05.sum())
 print(" num predicted_as_not_positive and central_pe>0.5:", index_negative_and_central_pe_gt_05.sum())
 print(" num predicted_as_not_positive and central_pe>0.5:", index_negative_and_rightsided_pe_gt_05.sum())
 print(" num predicted_as_not_positive and leftsided_pe>0.5:", index_negative_and_leftsided_pe_gt_05.sum())
 print(" num predicted_as_not_positive and chronic_pe>0.5:", index_negative_and_chronic_pe_gt_05.sum())
 print(" num predicted_as_not_positive and acute_and_chronic_pe>0.5:", index_negative_and_acute_and_chronic_pe_gt_05.sum())
 print("process positive case")
 print(" num predicted_as_positive and rv_lv_ratio_lt_1<=rv_lv_ratio_gte_1:", index_positive_and_rv_gte_lv.sum())
 print(" num predicted_as_positive and rv_lv_ratio_lt_1>rv_lv_ratio_gte_1:", index_positive_and_rv_lt_lv.sum())
 print(" num predicted_as_positive and(rv_lv_ratio_lt_1<=rv_lv_ratio_gte_1)and(rv_lv_ratio_gte_1<=0.5): ",
index_positive_and_rv_gte_lv_and_rv_lv_ratio_gte_1_lte_05.sum())
 print(" num predicted_as_positive and(rv_lv_ratio_lt_1<=rv_lv_ratio_gte_1)and rv_lv_ratio_lt_1>0.5: ",
index_positive_and_rv_gte_lv_and_rv_lv_ratio_lt_1_gt_05.sum())
 print(" num predicted_as_positive and(rv_lv_ratio_lt_1>rv_lv_ratio_gte_1)and rv_lv_ratio_lt_1<=0.5: ",
index_positive_and_rv_lt_lv_and_rv_lv_ratio_lt_1_lte_05.sum())
 print(" num predicted_as_positive and(rv_lv_ratio_lt_1>rv_lv_ratio_gte_1)and rv_lv_ratio_gte_1>0.5: ",
index_positive_and_rv_lt_lv_and_rv_lv_ratio_gte_1_gt_05.sum())
 print(" num predicted_as_positive and central is greatest:", index_positive_and_central_is_greatest.sum())
 print(" num predicted_as_positive and right is greatest:", index_positive_and_right_is_greatest.sum())
 print(" num predicted_as_positive and left is greatest:", index_positive_and_left_is_greatest.sum())
 print(" num predicted_as_positive and central is greatest and central_pe<=0.5:", index_positive_and_central_is_greatest_and_central_pe_lte_05.sum())
 print(" num predicted_as_positive and right is greatest and rightsided_pe<=0.5:", index_positive_and_right_is_greatest_and_rightsided_pe_lte_05.sum())
 print(" num predicted_as_positive and left is greatest and leftsided_pe<=0.5:", index_positive_and_left_is_greatest_and_leftsided_pe_lte_05.sum())
 print(" num both chronic_pe and acute_and_chronic_pe is positive:", index_double_positive.sum())
 print(" num both chronic_pe and acute_and_chronic_pe is positive and chronic<=acute_and_chronic:", index_double_positive_and_chronic_lte_acute_and_chronic.sum())
 print(" num both chronic_pe and acute_and_chronic_pe is positive and chronic>acute_and_chronic:", index_double_positive_and_chronic_gt_acute_and_chronic.sum())
 print("process image level")
 print(" num img of predicted_as_positive:", index_positive_i.sum())
 print(" num img of predicted_as_negative:",(index_positive_i==0 ).sum())
 print(" num img of peak:", df_pred2['peak'].sum())
 print(" num img of predicted_as_negative and pe_present_on_image>0.5:", index_negative_and_pe_present_on_image_gt_05_i.sum())
 print(" num img of predicted_as_positive and peak and pe_present_on_image<=0.5:", index_positive_and_peak_and_pe_present_on_image_lte_05_i.sum())
 return df_pred_s, df_pred
train_df['cast'].fillna(0 ).apply(lambda x: len(x)if x != 0 else 0 ).value_counts()<count_missing_values>
df_pred_study_const, df_pred_image_const = solve_conflict(df_pred_study, df_pred_image)
 df_pred_study_const.head()
for i in train_df.columns:
num_nas = train_df[i].isnull().sum()
if num_nas > 0:
print(i, " :", num_nas )<count_missing_values>
df_sub_pred = copy.deepcopy(df_pred_image_const[[col_index, col_targets[-1]]])
 df_sub_pred.columns = ['id', 'label']
for i, col in enumerate(col_targets[:-1]):
df_tmp = df_pred_study_const[[col_groupby, col]]
df_tmp.columns = ['id', 'label']
df_tmp['id'] = df_tmp['id'] + '_{}'.format(col)
 df_sub_pred = pd.concat([df_sub_pred, df_tmp])
 df_sub_pred = df_sub_pred.reset_index(drop=True)
 print(df_sub_pred.shape)
 df_sub_pred.head()
for i in test_df.columns:
num_nas = test_df[i].isnull().sum()
if num_nas > 0:
print(i, " :", num_nas )<count_values>
df_sub = pd.read_csv(df_sub_path)
 print(df_sub.shape)
 df_sub.head()
test_df['status'].value_counts()<feature_engineering>
df_sub = pd.merge(df_sub[['id']], df_sub_pred, on='id', how='left')
 print(df_sub.shape)
 df_sub.head()
train_df['runtime'] = train_df['runtime'].fillna(train_df['runtime'].median())
 test_df['runtime'] = test_df['runtime'].fillna(test_df['runtime'].median())
 train_df['tagline'] = train_df['tagline'].fillna('')
 test_df['tagline'] = test_df['tagline'].fillna('')
 train_df['homepage'] = train_df['homepage'].fillna(0)
 test_df['homepage'] = test_df['homepage'].fillna(0)
 train_df['has_tagline'] = train_df['tagline'].apply(lambda x: 1 if x != '' else 0)
 test_df['has_tagline'] = test_df['tagline'].apply(lambda x: 1 if x != '' else 0)
 train_df['has_homepage'] = train_df['homepage'].apply(lambda x: 1 if x != 0 else 0)
 test_df['has_homepage'] = test_df['homepage'].apply(lambda x: 1 if x != 0 else 0)
 train_df['budget'] = train_df['budget'].apply(lambda x: x if x > 0 else np.NaN)
 test_df['budget'] = test_df['budget'].apply(lambda x: x if x > 0 else np.NaN)
 train_df['budget'] = train_df['budget'].fillna(train_df['budget'].median())
 test_df['budget'] = test_df['budget'].fillna(test_df['budget'].median())
 test_df['status'] = test_df['status'].fillna(test_df['status'].mode() [0])
<categorify>
mean_targets = [
0.674681,
0.021569,
0.040115,
0.019920,
0.055090,
0.212117,
0.257590,
0.129139,
0.174612,
0.289885,
]
df_sub_mean = copy.deepcopy(df_test_full[[col_index]])
 df_sub_mean.columns = ['id']
df_sub_mean['label'] = mean_targets[-1]
for i, col in enumerate(col_targets[:-1]):
df_tmp = df_test_full_study[[col_groupby]]
df_tmp.columns = ['id']
df_tmp['label'] = mean_targets[i]
df_tmp['id'] = df_tmp['id'] + '_{}'.format(col)
 df_sub_mean = pd.concat([df_sub_mean, df_tmp])
 df_sub_mean = df_sub_mean.reset_index(drop=True)
 print(df_sub_mean.shape )
le = LabelEncoder()
train_df['status'] = le.fit_transform(train_df['status'])
 test_df['status'] = le.fit_transform(test_df['status'])
 train_df['status'].value_counts()<feature_engineering>
df_sub['label'][pd.isna(df_sub['label'])] = pd.merge(df_sub[['id']], df_sub_mean, on='id', how='left')['label'][pd.isna(df_sub['label'])]
df_sub.head(20 )
train_df['log_revenue'] = np.log1p(train_df['revenue'])
 train_df['log_budget'] = np.log1p(train_df['budget'])
 test_df['log_budget'] = np.log1p(test_df['budget'] )<train_model>
def check_consistency2(df_exam, df_image, test):

 df = df_exam.merge(df_image, how = 'left', on = 'StudyInstanceUID')
 ids = ['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']
labels = [c for c in df.columns if c not in ids]
df = df[ids + labels]
df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID'] ).pe_present_on_image.max())
 df_pos = df.loc[df.positive_images_in_exam > 0.5]
df_neg = df.loc[df.positive_images_in_exam <= 0.5]
rule1a = df_pos.loc[(( df_pos.rv_lv_ratio_lt_1 > 0.5)& 
(df_pos.rv_lv_ratio_gte_1 > 0.5)) | 
(( df_pos.rv_lv_ratio_lt_1 <= 0.5)& 
(df_pos.rv_lv_ratio_gte_1 <= 0.5)) ].reset_index(drop = True)
 rule1a['broken_rule'] = '1a'
rule1b = df_pos.loc[(df_pos.central_pe <= 0.5)& 
(df_pos.rightsided_pe <= 0.5)& 
(df_pos.leftsided_pe <= 0.5)].reset_index(drop = True)
 rule1b['broken_rule'] = '1b'
rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5)& 
(df_pos.chronic_pe > 0.5)].reset_index(drop = True)
 rule1c['broken_rule'] = '1c'
rule1d = df_pos.loc[(df_pos.indeterminate > 0.5)| 
(df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)
 rule1d['broken_rule'] = '1d'
rule2a = df_neg.loc[(( df_neg.indeterminate > 0.5)& 
(df_neg.negative_exam_for_pe > 0.5)) | 
(( df_neg.indeterminate <= 0.5)& 
(df_neg.negative_exam_for_pe <= 0.5)) ].reset_index(drop = True)
 rule2a['broken_rule'] = '2a'
rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1 > 0.5)| 
(df_neg.rv_lv_ratio_gte_1 > 0.5)| 
(df_neg.central_pe > 0.5)| 
(df_neg.rightsided_pe > 0.5)| 
(df_neg.leftsided_pe > 0.5)| 
(df_neg.acute_and_chronic_pe > 0.5)| 
(df_neg.chronic_pe > 0.5)].reset_index(drop = True)
 rule2b['broken_rule'] = '2b'
errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)
 print('Found', len(errors), 'inconsistent predictions')
 return errors
vectorizer = TfidfVectorizer(
 sublinear_tf=True,
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
min_df=5)
 overview_text = vectorizer.fit_transform(train_df['overview'].fillna(''))
linreg = LinearRegression()
linreg.fit(overview_text, train_df['log_revenue'])
 eli5.show_weights(linreg, vec=vectorizer, top=20, feature_filter=lambda x: x != '<BIAS>' )<define_variables>
error = check_consistency2(df_pred_study_const, df_pred_image_const, df_test )
train_texts = train_df[['title', 'tagline', 'overview', 'original_title']]
test_texts = test_df[['title', 'tagline', 'overview', 'original_title']]
train_df[['title', 'tagline', 'overview', 'original_title']].head()<feature_engineering>
if len(error)==0:
df_sub.to_csv('submission.csv', index=None)
 else:
print("error!" )