Kaggle competition: How Much Did It Rain? II
correlations = train.corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]
correlations.tail(10 )<define_variables>
INPUT_WIDTH = 19
N_FEATURES = 22
THRESHOLD = 73
feature_names = [x for x in train.columns if x not in ['ID_code','target']]<train_model>
train_df = pd.read_csv(".. /input/train.csv")
 train_df[train_df.columns[1:]] = train_df[train_df.columns[1:]].astype(np.float32)
 train_df.head(20 )
model = CatBoostClassifier(iterations=5000,learning_rate=0.01,eval_metric='AUC',random_seed=42,verbose=False,task_type='GPU')
 model.fit(train[feature_names],target)
 pred1 = model.predict_proba(test[feature_names] )<predict_on_test>
train_new = train_new.fillna(0.0)
 train_new = train_new.reset_index(drop=True)
 train_new.head()
model = lgb.LGBMClassifier(n_estimators=5000,n_jobs = -1,learning_rate=0.01)
 model.fit(train[feature_names],target)
 pred2 = model.predict_proba(test[feature_names] )<predict_on_test>
df_temp = pd.DataFrame(train_new.groupby('Id')['Expected'].mean())
 meaningful_ids = np.array(df_temp[df_temp['Expected'] < THRESHOLD].index)
 del df_temp
train_final = train_new[np.in1d(train_new.Id, meaningful_ids)]
del train_new, meaningful_ids
train_final.shape
model = xgb.XGBClassifier(n_estimators=5000,n_jobs = -1,learning_rate=0.01)
 model.fit(train[feature_names],target)
 pred3 = model.predict_proba(test[feature_names] )<predict_on_test>
train_gp = train_final.groupby("Id")
 train_size = len(train_gp)
 del train_final
X_train = np.zeros(( train_size, INPUT_WIDTH, N_FEATURES), dtype=np.float32)
 y_train = np.zeros(train_size, dtype=np.float32)
 seq_len_train = np.zeros(train_size, dtype=np.float32)
 i = 0
for _, group in train_gp:
X = group.values
seq_len = X.shape[0]
X_train[i,:seq_len,:] = X[:,1:23]
y_train[i] = X[0,23]
seq_len_train[i] = seq_len
i += 1
del X
del train_gp
X_train.shape, y_train.shape
model = RandomForestClassifier(n_estimators=1000,n_jobs = -1)
 model.fit(train[feature_names],target)
 pred4 = model.predict_proba(test[feature_names] )<compute_test_metric>
test_df = pd.read_csv(".. /input/test.csv")
 test_df[test_df.columns[1:]] = test_df[test_df.columns[1:]].astype(np.float32)
 test_ids = np.array(test_df.Id.unique())
 test_final = test_df.fillna(0.0)
 test_final = test_final.reset_index(drop=True)
 del test_df
pred = []
for i in range(len(pred1)) :
pred.append(( pred1[i][1] + pred2[i][1] + pred3[i][1] + pred4[i][1])/4 )<save_to_csv>
test_gp = test_final.groupby("Id")
 test_size = len(test_gp)
 del test_final
X_test = np.zeros(( test_size, INPUT_WIDTH, N_FEATURES), dtype=np.float32)
 seq_len_test = np.zeros(test_size, dtype=np.float32)
 i = 0
for _, group in test_gp:
X = group.values
seq_len = X.shape[0]
X_test[i,:seq_len,:] = X[:,1:23]
seq_len_test[i] = seq_len
i += 1
del X
del test_gp
X_test.shape
sub = pd.DataFrame()
sub['ID_code'] = test['ID_code']
sub['target'] = pred
sub.to_csv('result.csv', index=False )<install_modules>
TimeDistributed, Flatten, Bidirectional
<import_modules>
es_callback = EarlyStopping(monitor='val_loss', min_delta=0, patience=5 )
Compose,
HorizontalFlip, ShiftScaleRotate,VerticalFlip,
RandomBrightness,RandomCrop,RandomContrast 
)
 test_images_dir = '.. /input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_test/'
train_images_dir = '.. /input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/' 
<set_options>
BATCH_SIZE = 1024
N_EPOCHS = 50
def _read(path, desired_size):

 dcm = pydicom.dcmread(path)
 try:
img = bsb_window(dcm,desired_size)
 except:
img = np.zeros(desired_size)
 return img
plt.imshow(
 _read(train_images_dir+'ID_5c8b5d701'+'.dcm',(256, 256)) , cmap=plt.cm.bone 
);<define_variables>
def get_model_simple(shape=(19,22)) :
inp = Input(shape)
 x = CuDNNLSTM(64, return_sequences=False )(inp)
 x = Dense(1 )(x)
 model = Model(inp, x)
 return model
class DataGenerator(keras.utils.Sequence):
def __init__(self, list_IDs, labels=None, batch_size=1, img_size=(512, 512, 1),
img_dir=train_images_dir,augment=None, *args, **kwargs):
self.list_IDs = list_IDs
self.labels = labels
self.batch_size = batch_size
self.img_size = img_size
self.img_dir = img_dir
self.on_epoch_end()
self.augment = augment
def __len__(self):
return int(ceil(len(self.indices)/ self.batch_size))
def __getitem__(self, index):
indices = self.indices[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_temp = [self.list_IDs[k] for k in indices]
if self.labels is not None:
X, Y = self.__data_generation(list_IDs_temp)
 return X, Y
else:
X = self.__data_generation(list_IDs_temp)
 return X
def on_epoch_end(self):
if self.labels is not None:
keep_prob = self.labels.iloc[:, 0].map({0: 0.3, 1: 1})
 keep =(keep_prob > np.random.rand(len(keep_prob)))
 self.indices = np.arange(len(self.list_IDs)) [keep]
np.random.shuffle(self.indices)
 else:
self.indices = np.arange(len(self.list_IDs))
def __data_generation(self, list_IDs_temp):
X = np.empty(( self.batch_size, *self.img_size))
if self.labels is not None:
Y = np.empty(( self.batch_size, 6), dtype=np.float32)
 for i, ID in enumerate(list_IDs_temp):
X[i,] = _read(self.img_dir+ID+".dcm", self.img_size)
 Y[i,] = self.labels.loc[ID].values
if self.augment:
X = self.__augment(X)
 return X, Y
else:
for i, ID in enumerate(list_IDs_temp):
X[i,] = _read(self.img_dir+ID+".dcm", self.img_size)
 return X
def __random_transform(self,img):
composition = Compose([
HorizontalFlip() ,
VerticalFlip() ,
ShiftScaleRotate(rotate_limit=45, shift_limit=0.15, scale_limit=0.15),
RandomBrightness() ,
RandomContrast()
])
 composed = composition(image=img)
 aug_img = composed['image']
return aug_img
def __augment(self,img_batch):
for i in range(img_batch.shape[0]):
img_batch[i, ] = self.__random_transform(img_batch[i, ])
 return img_batch<compute_train_metric>
model_0.fit(X_train, y_train,
batch_size=BATCH_SIZE, epochs=N_EPOCHS,
validation_split=0.2, callbacks=[es_callback] )
def weighted_log_loss(y_true, y_pred):

 class_weights = np.array([2., 1., 1., 1., 1., 1.])
 eps = K.epsilon()
y_pred = K.clip(y_pred, eps, 1.0-eps)
 out = -(y_true * K.log(y_pred)* class_weights
+(1.0 - y_true)* K.log(1.0 - y_pred)* class_weights)
 return K.mean(out, axis=-1)
 def _normalized_weighted_average(arr, weights=None):

 if weights is not None:
scl = K.sum(weights)
 weights = K.expand_dims(weights, axis=1)
 return K.sum(K.dot(arr, weights), axis=1)/ scl
return K.mean(arr, axis=1)
 def weighted_loss(y_true, y_pred):

 class_weights = K.variable([2., 1., 1., 1., 1., 1.])
 eps = K.epsilon()
y_pred = K.clip(y_pred, eps, 1.0-eps)
 loss = -(y_true * K.log(y_pred)
 +(1.0 - y_true)* K.log(1.0 - y_pred))
loss_samples = _normalized_weighted_average(loss, class_weights)
 return K.mean(loss_samples)
 def weighted_log_loss_metric(trues, preds):

 class_weights = [2., 1., 1., 1., 1., 1.]
epsilon = 1e-7
preds = np.clip(preds, epsilon, 1-epsilon)
 loss = trues * np.log(preds)+(1 - trues)* np.log(1 - preds)
 loss_samples = np.average(loss, axis=1, weights=class_weights)
 return - loss_samples.mean()
def f1_score(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 recall = true_positives /(possible_positives + K.epsilon())
 return(2*precision*recall)/(precision+recall+ K.epsilon())
<choose_model_class>
y_pred_0 = model_0.predict(X_test)
 submission_0 = pd.DataFrame({'Id': test_ids, 'Expected': y_pred_0.reshape(-1)})
 submission_0.to_csv('submission_0.csv', index=False )
class PredictionCheckpoint(keras.callbacks.Callback):
def __init__(self, test_df, valid_df,
test_images_dir=test_images_dir,
valid_images_dir=train_images_dir,
batch_size=32, input_size=(224, 224, 3)) :
self.test_df = test_df
self.valid_df = valid_df
self.test_images_dir = test_images_dir
self.valid_images_dir = valid_images_dir
self.batch_size = batch_size
self.input_size = input_size
def on_train_begin(self, logs={}):
self.test_predictions = []
self.valid_predictions = []
def on_epoch_end(self,batch, logs={}):
self.test_predictions.append(
 self.model.predict_generator(
 DataGenerator(self.test_df.index, None, self.batch_size, self.input_size, self.test_images_dir), verbose=2)[:len(self.test_df)])
 class MyDeepModel:
def __init__(self, engine, input_dims, batch_size=5, num_epochs=4, learning_rate=1e-3,
decay_rate=1.0, decay_steps=1, weights="imagenet", verbose=1):
self.engine = engine
self.input_dims = input_dims
self.batch_size = batch_size
self.num_epochs = num_epochs
self.learning_rate = learning_rate
self.decay_rate = decay_rate
self.decay_steps = decay_steps
self.weights = weights
self.verbose = verbose
self._build()
def _build(self):
engine = self.engine(include_top=False, weights=self.weights, input_shape=self.input_dims,
backend = keras.backend, layers = keras.layers,
models = keras.models, utils = keras.utils)
 x = keras.layers.GlobalAveragePooling2D(name='avg_pool' )(engine.output)
 x = keras.layers.Dropout(0.3 )(x)
 out = keras.layers.Dense(6, activation="sigmoid", name='dense_output' )(x)
 self.model = keras.models.Model(inputs=engine.input, outputs=out)
 self.model.compile(loss='binary_crossentropy', optimizer=keras.optimizers.Nadam() , metrics=[weighted_loss,f1_score])
 def fit_and_predict(self, train_df, valid_df, test_df):
pred_history = PredictionCheckpoint(test_df, valid_df, input_size=self.input_dims)
 scheduler = keras.callbacks.LearningRateScheduler(lambda epoch: self.learning_rate * pow(self.decay_rate, floor(epoch / self.decay_steps)))
 history = self.model.fit_generator(
 DataGenerator(
 train_df.index,
train_df,
self.batch_size,
self.input_dims,
train_images_dir,
augment=None 
),
epochs=self.num_epochs,
verbose=self.verbose,
use_multiprocessing=True,
workers=4,
callbacks=[scheduler] 
)
 return history
def save(self, path):
self.model.save_weights(path)
 def load(self, path):
self.model.load_weights(path )<load_from_csv>
def get_model_seq(shape=(19,22)) :
inp = Input(shape)
 x = CuDNNLSTM(64, return_sequences=True )(inp)
 x = TimeDistributed(Dense(10))(x)
 x = Flatten()(x)
 x = Dense(1 )(x)
 model = Model(inp, x)
 return model
def read_testset(filename=".. /input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_sample_submission.csv"):
df = pd.read_csv(filename)
 df["Image"] = df["ID"].str.slice(stop=12)
 df["Diagnosis"] = df["ID"].str.slice(start=13)
 df = df.loc[:, ["Label", "Diagnosis", "Image"]]
df = df.set_index(['Image', 'Diagnosis'] ).unstack(level=-1)
 return df
def read_trainset(filename=".. /input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train.csv"):
df = pd.read_csv(filename)
 df["Image"] = df["ID"].str.slice(stop=12)
 df["Diagnosis"] = df["ID"].str.slice(start=13)

df = df[df.duplicated() ==False]
df = df.reset_index(drop=True)
 df = df.loc[:, ["Label", "Diagnosis", "Image"]]
df = df.set_index(['Image', 'Diagnosis'] ).unstack(level=-1)
 return df
test_df = read_testset()
df = read_trainset()<normalization>
model_1.fit(X_train, y_train,
batch_size=BATCH_SIZE, epochs=N_EPOCHS,
validation_split=0.2, callbacks=[es_callback] )
1/K.epsilon()<train_model>
y_pred_1 = model_1.predict(X_test)
 submission_1 = pd.DataFrame({'Id': test_ids, 'Expected': y_pred_1.reshape(-1)})
 submission_1.to_csv('submission_1.csv', index=False )
ss = ShuffleSplit(n_splits=10, test_size=0.1, random_state=42 ).split(df.index)
 train_idx, valid_idx = next(ss)
 model = MyDeepModel(engine=DenseNet121, input_dims=(224, 224, 3), batch_size=32, learning_rate=1e-3,
num_epochs=5, decay_rate=0.8, decay_steps=1, weights="imagenet", verbose=1)
 history = model.fit_and_predict(df.iloc[train_idx], df.iloc[valid_idx], test_df)
<randomize_order>
def get_model_bilstm(shape=(19,22)) :
inp = Input(shape)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(inp)
 x = TimeDistributed(Dense(10))(x)
 x = Flatten()(x)
 x = Dense(1 )(x)
 model = Model(inp, x)
 return model
keep_prob = df.iloc[train_idx].iloc[:, 0].map({0: 0.3, 1: 1})
 keep =(keep_prob > np.random.rand(len(keep_prob)) )<count_values>
model_2.fit(X_train, y_train,
batch_size=BATCH_SIZE, epochs=N_EPOCHS,
validation_split=0.2, callbacks=[es_callback] )
df.iloc[train_idx][keep]['Label']['any'].value_counts()<save_to_csv>
y_pred_2 = model_2.predict(X_test)
 submission_2 = pd.DataFrame({'Id': test_ids, 'Expected': y_pred_2.reshape(-1)})
 submission_2.to_csv('submission_2.csv', index=False )
prediction = model.model.predict_generator(DataGenerator(test_df.index, None, 32,(224, 224, 3), test_images_dir), verbose=1)
 test_df_2 = test_df.copy()
test_df.iloc[:, :] = prediction[:len(test_df)]
test_df = test_df.stack().reset_index()
test_df.insert(loc=0, column='ID', value=test_df['Image'].astype(str)+ "_" + test_df['Diagnosis'])
 test_df = test_df.drop(["Image", "Diagnosis"], axis=1)
 test_df.to_csv('submission.csv', index=False)
<install_modules>
def get_model_deep(shape=(19,22)) :
inp = Input(shape)
 x = Dense(16 )(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = TimeDistributed(Dense(64))(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = TimeDistributed(Dense(1))(x)
 x = AveragePooling1D()(x)
 x = Flatten()(x)
 x = Dense(1 )(x)
 model = Model(inp, x)
 return model
!pip install git+https://github.com/fastai/fastai_dev<define_variables>
model_3.fit(X_train, y_train,
batch_size=BATCH_SIZE, epochs=N_EPOCHS,
validation_split=0.2, callbacks=[es_callback] )
dir_csv = '.. /input/rsna-intracranial-hemorrhage-detection'
dir_train_img = '.. /input/rsna-train-stage-1-images-png-224x/stage_1_train_png_224x'
dir_test_img = '.. /input/rsna-test-stage-1-images-png-224x/stage_1_test_png_224x'
dir_train_csv = ".. /input/rsnsa-intracranial-hemorrahage-detection/stage_1_train.csv"
dir_sample_submission_csv = ".. /input/rsnsa-intracranial-hemorrahage-detection/stage_1_sample_submission.csv"<import_modules>
y_pred_3 = model_3.predict(X_test)
 submission_3 = pd.DataFrame({'Id': test_ids, 'Expected': y_pred_3.reshape(-1)})
 submission_3.to_csv('submission_3.csv', index=False )
<define_variables><EOS> 
y_pred_avg =(y_pred_0 + y_pred_1 + y_pred_2 + y_pred_3)/ 4
submission_avg = pd.DataFrame({'Id': test_ids, 'Expected': y_pred_avg.reshape(-1)})
 submission_avg.to_csv('submission_avg.csv', index=False )