Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<import_modules>
import numpy as np
import pandas as pd
from xgboost import XGBRegressor
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import RandomizedSearchCV,cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn.metrics import make_scorer
from sklearn.ensemble import RandomForestRegressor
from itertools import product
import re
import nltk
from nltk import word_tokenize
from nltk.corpus import stopwords
import os
import sys
import random
import math
import numpy as np
import cv2
import matplotlib.pyplot as plt
import json
import pydicom
from imgaug import augmenters as iaa
from tqdm import tqdm
import pandas as pd
import glob<define_variables>
!pip install pymorphy2
!pip install pymorphy2-dicts
!pip install DAWG-Python
DATA_DIR = '/kaggle/input/rsna-pneumonia-detection-challenge/'
ROOT_DIR = '/kaggle/working'<load_from_zip>
import pymorphy2
!git clone https://www.github.com/matterport/Mask_RCNN.git
os.chdir('Mask_RCNN')
<define_variables>
sample_submissions=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sample_submission.csv")
 sales_train_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sales_train.csv")
 test_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 items_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/items.csv")
 shops_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/shops.csv")
 item_categories_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/item_categories.csv" )
train_dicom_dir = os.path.join(DATA_DIR, 'stage_2_train_images')
 test_dicom_dir = os.path.join(DATA_DIR, 'stage_2_test_images' )<create_dataframe>
print(sales_train_df["shop_id"].nunique())
 print(test_df["shop_id"].nunique() )
def get_dicom_fps(dicom_dir):
dicom_fps = glob.glob(dicom_dir+'/'+'*.dcm')
 return list(set(dicom_fps))
def parse_dataset(dicom_dir, anns):
image_fps = get_dicom_fps(dicom_dir)
 image_annotations = {fp: [] for fp in image_fps}
for index, row in anns.iterrows() :
fp = os.path.join(dicom_dir, row['patientId']+'.dcm')
 image_annotations[fp].append(row)
 return image_fps, image_annotations<init_hyperparams>
print(sales_train_df["item_id"].nunique())
 print(test_df["item_id"].nunique() )
class DetectorConfig(Config):

 NAME = 'pneumonia'
GPU_COUNT = 1
IMAGES_PER_GPU = 8
BACKBONE = 'resnet50'
NUM_CLASSES = 2
IMAGE_MIN_DIM = 256
IMAGE_MAX_DIM = 256
RPN_ANCHOR_SCALES =(32, 64, 128, 256)
 TRAIN_ROIS_PER_IMAGE = 32
MAX_GT_INSTANCES = 3
DETECTION_MAX_INSTANCES = 3
DETECTION_MIN_CONFIDENCE = 0.7
DETECTION_NMS_THRESHOLD = 0.1
STEPS_PER_EPOCH = 200
config = DetectorConfig()
config.display()<categorify>
nltk.download("stopwords" )
class DetectorDataset(utils.Dataset):

 def __init__(self, image_fps, image_annotations, orig_height, orig_width):
super().__init__(self)
 self.add_class('pneumonia', 1, 'Lung Opacity')
 for i, fp in enumerate(image_fps):
annotations = image_annotations[fp]
self.add_image('pneumonia', image_id=i, path=fp,
annotations=annotations, orig_height=orig_height, orig_width=orig_width)
 def image_reference(self, image_id):
info = self.image_info[image_id]
return info['path']
def load_image(self, image_id):
info = self.image_info[image_id]
fp = info['path']
ds = pydicom.read_file(fp)
 image = ds.pixel_array
if len(image.shape)!= 3 or image.shape[2] != 3:
image = np.stack(( image,)* 3, -1)
 return image
def load_mask(self, image_id):
info = self.image_info[image_id]
annotations = info['annotations']
count = len(annotations)
 if count == 0:
mask = np.zeros(( info['orig_height'], info['orig_width'], 1), dtype=np.uint8)
 class_ids = np.zeros(( 1,), dtype=np.int32)
 else:
mask = np.zeros(( info['orig_height'], info['orig_width'], count), dtype=np.uint8)
 class_ids = np.zeros(( count,), dtype=np.int32)
 for i, a in enumerate(annotations):
if a['Target'] == 1:
x = int(a['x'])
 y = int(a['y'])
 w = int(a['width'])
 h = int(a['height'])
 mask_instance = mask[:, :, i].copy()
cv2.rectangle(mask_instance,(x, y),(x+w, y+h), 255, -1)
 mask[:, :, i] = mask_instance
class_ids[i] = 1
return mask.astype(np.bool), class_ids.astype(np.int32 )<load_from_csv>
def text_preprocessing(text):
txt=""
morph=pymorphy2.MorphAnalyzer()
stop_words_ru=stopwords.words("russian")
 stop_words_eng=stopwords.words("english")
 stroka=re.sub('[^A-Za-zА-Яа-я]+',' ',text)
 strs=stroka.split(" ")
 filtered_strs=[morph.parse(w.lower())[0].normal_form for w in strs if(w not in stop_words_ru and w not in stop_words_eng)]
txt=" ".join(filtered_strs)
 return txt
anns = pd.read_csv(os.path.join(DATA_DIR, 'stage_2_train_labels.csv'))
anns.head()<create_dataframe>
image_fps, image_annotations = parse_dataset(train_dicom_dir, anns=anns )<load_from_csv>
sales_train_df=sales_train_df.join(shops_df.set_index("shop_id"),on="shop_id",how="left" ).join(
 items_df.set_index("item_id"),on="item_id",how="left" ).join(item_categories_df.set_index("item_category_id")
 ,on="item_category_id",how="left")
 sales_train_df.head().T
ds = pydicom.read_file(image_fps[0])
 image = ds.pixel_array<define_variables>
test_df=test_df.join(shops_df.set_index("shop_id"),on="shop_id",how="left" ).join(
 items_df.set_index("item_id"),on="item_id",how="left" ).join(item_categories_df.set_index("item_category_id")
 ,on="item_category_id",how="left")
 test_df.head().T
ORIG_SIZE = 1024<load_from_csv>
sales_train_df[sales_train_df.item_cnt_day>2000]
test_preds = pd.read_csv('/kaggle/input/stage-1-ids/stage_1_sample_submission.csv' )<define_variables>
sales_train_df[sales_train_df.item_category_id==9]["item_cnt_day"].mean()
test_patient_ids = set(test_preds.patientId )<split>
sales_train_df[(sales_train_df.shop_id==12)&(sales_train_df.item_id==11373)]["item_cnt_day"].mean()
image_fps_list = list(image_fps)
 random.seed(42)
 random.shuffle(image_fps_list)
 val_size = []
val_size = 1500
image_fps_val = image_fps_list[:val_size]
image_fps_train = image_fps_list[val_size:]
print(len(image_fps_train), len(image_fps_val))<create_dataframe>
sales_train_df=sales_train_df[~sales_train_df.isin(sales_train_df[sales_train_df.item_cnt_day>2000])]
dataset_train = DetectorDataset(image_fps_train, image_annotations, ORIG_SIZE, ORIG_SIZE)
 dataset_train.prepare()<create_dataframe>
sales_train_df[sales_train_df.item_price>50000]
dataset_val = DetectorDataset(image_fps_val, image_annotations, ORIG_SIZE, ORIG_SIZE)
 dataset_val.prepare()<load_pretrained>
sales_train_df=sales_train_df[~sales_train_df.isin(sales_train_df[sales_train_df.item_price>50000])]
model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)
 model.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[
"mrcnn_class_logits", "mrcnn_bbox_fc",
"mrcnn_bbox", "mrcnn_mask"] )<init_hyperparams>
sales_train_df[sales_train_df.item_cnt_day>800]
LEARNING_RATE = 0.005
warnings.filterwarnings("ignore" )<train_model>
sales_train_df[(sales_train_df.shop_id==12)&(sales_train_df.item_id==20949)]["item_cnt_day"].mean()
%%time
model.train(dataset_train, dataset_val,
learning_rate=LEARNING_RATE*2,
epochs=2,
layers='heads',
augmentation=None )<train_model>
sales_train_df=sales_train_df[~sales_train_df.isin(sales_train_df[sales_train_df.item_cnt_day>800])]
%%time
model.train(dataset_train, dataset_val,
learning_rate=LEARNING_RATE,
epochs=7,
layers='all',
augmentation=augmentation )<train_model>
test_unique_items=test_df[~test_df["item_id"].isin(item_id_uniq)]
test_unique_items.head()
%%time
model.train(dataset_train, dataset_val,
learning_rate=LEARNING_RATE/5,
epochs=15,
layers='all',
augmentation=augmentation )<load_pretrained>
test_unique_items["item_id"].nunique()
class InferenceConfig(DetectorConfig):
GPU_COUNT = 1
IMAGES_PER_GPU = 1
inference_config = InferenceConfig()
model = modellib.MaskRCNN(mode='inference',
config=inference_config,
model_dir=ROOT_DIR)
 assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
 model.load_weights(model_path, by_name=True )<find_best_params>
%%time
train=[]
col=["date_block_num","shop_id","item_id"]
for i in range(34):
sales=sales_train_df[sales_train_df.date_block_num==i]
train.append(np.array(list(product([i],sales.shop_id.unique() ,sales.item_id.unique())) ,dtype="int16"))
train=pd.DataFrame(np.vstack(train),columns=col)
 train["date_block_num"]=train["date_block_num"].astype(np.int8)
 train["shop_id"]=train["shop_id"].astype(np.int8)
 train["item_id"]=train["item_id"].astype(np.int16)
 print(train.head() )
test_image_fps = get_dicom_fps(test_dicom_dir )<save_to_csv>
month_df=sales_train_df[["date_block_num","shop_id","item_id","item_cnt_day"]].groupby(["date_block_num","shop_id","item_id"] ).sum()
month_df.head()
def predict(image_fps, filepath='submission.csv', min_conf=0.95):
resize_factor = ORIG_SIZE / config.IMAGE_SHAPE[0]
with open(filepath, 'w')as file:
file.write("patientId,PredictionString
")
 for image_id in tqdm(image_fps):
ds = pydicom.read_file(image_id)
 image = ds.pixel_array
if len(image.shape)!= 3 or image.shape[2] != 3:
image = np.stack(( image,)* 3, -1)
 image, window, scale, padding, crop = utils.resize_image(
 image,
min_dim=config.IMAGE_MIN_DIM,
min_scale=config.IMAGE_MIN_SCALE,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
 patient_id = os.path.splitext(os.path.basename(image_id)) [0]
results = model.detect([image])
 r = results[0]
out_str = ""
out_str += patient_id
out_str += ","
assert(len(r['rois'])== len(r['class_ids'])== len(r['scores']))
if len(r['rois'])== 0:
pass
else:
num_instances = len(r['rois'])
 for i in range(num_instances):
if r['scores'][i] > min_conf:
out_str += ' '
out_str += str(round(r['scores'][i], 2))
out_str += ' '
x1 = r['rois'][i][1]
y1 = r['rois'][i][0]
width = r['rois'][i][3] - x1
height = r['rois'][i][2] - y1
bboxes_str = "{} {} {} {}".format(x1*resize_factor, y1*resize_factor, \
width*resize_factor, height*resize_factor)
 out_str += bboxes_str
file.write(out_str+"
" )<predict_on_test>
date_block_total["year"]=date_block_total.date_block_num.apply(lambda x:(( x//12)+2013))
date_block_total["month"]=date_block_total.date_block_num.apply(lambda x:(x%12))
date_block_total.head()
submission_fp = os.path.join(ROOT_DIR, 'submission.csv')
 predict(test_image_fps, filepath=submission_fp)
 print(submission_fp )<load_from_csv>
train["year"]=train.date_block_num.apply(lambda x:(( x//12)+2013))
train["month"]=train.date_block_num.apply(lambda x:(x%12))
train["year"]=train["year"].astype(np.int16)
 train["month"]=train["month"].astype(np.int8)
 train.head()
output = pd.read_csv(submission_fp, names=['patientId', 'PredictionString'])
 output.head(60 )<predict_on_test>
test_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 test_df["date_block_num"]=34
test_df["year"]=(( 34//12)+2013)
 test_df["month"]=(34%12)
 test_df["date_block_num"]=test_df["date_block_num"].astype(np.int8)
 test_df["shop_id"]=test_df["shop_id"].astype(np.int8)
 test_df["item_id"]=test_df["item_id"].astype(np.int16)
 test_df["year"]=test_df["year"].astype(np.int16)
 test_df["month"]=test_df["month"].astype(np.int8)
 test_df.head()
submission_val_fp = os.path.join(ROOT_DIR, 'coco-mask-rcnn-with-submission-0.166-retrain-validation.csv')
 predict(image_fps_val, filepath=submission_val_fp)
 print(submission_val_fp )<drop_column>
categorical=["year","month"]
!rm -rf /kaggle/working/Mask_RCNN<import_modules>
train=train.join(month_df,on=["date_block_num","shop_id","item_id"],how="left")
 train["item_cnt_day"]=train["item_cnt_day"].fillna(0 ).clip(0,20 ).astype(np.float16)
 train.head()
import os
import csv
import random
import pydicom
import numpy as np
import pandas as pd
from skimage import measure
from skimage.transform import resize
import tensorflow as tf
from tensorflow import keras
from matplotlib import pyplot as plt<load_from_csv>
train=pd.concat([train,test_df],ignore_index=True,sort=False,keys=col)
 train.fillna(0,inplace=True)
 train.head()
pneumonia_locations = {}
with open(os.path.join('.. /input/stage_1_train_labels.csv'), mode='r')as infile:
reader = csv.reader(infile)
 next(reader, None)
 for rows in reader:
filename = rows[0]
location = rows[1:5]
pneumonia = rows[5]
if pneumonia == '1':
location = [int(float(i)) for i in location]
if filename in pneumonia_locations:
pneumonia_locations[filename].append(location)
 else:
pneumonia_locations[filename] = [location]<define_variables>
def create_lag_features(df,feature):
numeric_features=[]
for i in range(1,7):
lagged=df.copy()
lagged=lagged[["date_block_num","shop_id","item_id",feature]]
lagged.columns=["date_block_num","shop_id","item_id",feature+"_lag_"+str(i)]
numeric_features.append(feature+"_lag_"+str(i))
lagged["date_block_num"]+=i
df=df.join(lagged.set_index(["date_block_num","shop_id","item_id"]),on=["date_block_num","shop_id","item_id"],how="left")
 return df,numeric_features
folder = '.. /input/stage_1_train_images'
filenames = os.listdir(folder)
 random.shuffle(filenames)
 n_valid_samples = 1284
train_filenames = filenames[n_valid_samples:]
valid_filenames = filenames[:n_valid_samples]
print('n train samples', len(train_filenames))
print('n valid samples', len(valid_filenames))
n_train_samples = len(filenames)- n_valid_samples<define_variables>
%%time
train,n_col=create_lag_features(train,"item_cnt_day")
 train=train.fillna(0)
 numeric=n_col
class generator(keras.utils.Sequence):
def __init__(self, folder, filenames, pneumonia_locations=None, batch_size=32, image_size=320, shuffle=True, augment=False, predict=False):
self.folder = folder
self.filenames = filenames
self.pneumonia_locations = pneumonia_locations
self.batch_size = batch_size
self.image_size = image_size
self.shuffle = shuffle
self.augment = augment
self.predict = predict
self.on_epoch_end()
def __load__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
msk = np.zeros(img.shape)
 filename = filename.split('.')[0]
if filename in pneumonia_locations:
for location in pneumonia_locations[filename]:
x, y, w, h = location
msk[y:y+h, x:x+w] = 1
if self.augment and random.random() > 0.5:
img = np.fliplr(img)
 msk = np.fliplr(msk)
 img = resize(img,(self.image_size, self.image_size), mode='reflect')
 msk = resize(msk,(self.image_size, self.image_size), mode='reflect')> 0.5
img = np.expand_dims(img, -1)
 msk = np.expand_dims(msk, -1)
 return img, msk
def __loadpredict__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
img = resize(img,(self.image_size, self.image_size), mode='reflect')
 img = np.expand_dims(img, -1)
 return img
def __getitem__(self, index):
filenames = self.filenames[index*self.batch_size:(index+1)*self.batch_size]
if self.predict:
imgs = [self.__loadpredict__(filename)for filename in filenames]
imgs = np.array(imgs)
 return imgs, filenames
else:
items = [self.__load__(filename)for filename in filenames]
imgs, msks = zip(*items)
 imgs = np.array(imgs)
 msks = np.array(msks)
 return imgs, msks
def on_epoch_end(self):
if self.shuffle:
random.shuffle(self.filenames)
 def __len__(self):
if self.predict:
return int(np.ceil(len(self.filenames)/ self.batch_size))
else:
return int(len(self.filenames)/ self.batch_size )<define_variables>
total_cnt_for_items=sales_train_df[["date_block_num","item_id","item_cnt_day"]].groupby(["date_block_num","item_id"] ).sum()
total_cnt_for_items.columns=["total_cnt_for_items"]
total_cnt_for_items.head()
BATCH_SIZE = 16
IMAGE_SIZE = 320<define_search_model>
train=train.join(total_cnt_for_items,on=["date_block_num","item_id"],how="left")
 train.fillna(0,inplace=True)
 train["total_cnt_for_items"]=train["total_cnt_for_items"].astype(np.float16)
 train.head().T
def create_downsample(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 1, padding='same', use_bias=False )(x)
 x = keras.layers.MaxPool2D(2 )(x)
 return x
def create_resblock(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 return keras.layers.add([x, inputs])
 def create_network(input_size, channels, n_blocks=2, depth=4):
inputs = keras.Input(shape=(input_size, input_size, 1))
x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(inputs)
 for d in range(depth):
channels = channels * 2
x = create_downsample(channels, x)
 for b in range(n_blocks):
x = create_resblock(channels, x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(256, 1, activation=None )(x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2DTranspose(128,(8,8),(4,4), padding="same", activation=None )(x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(1, 1, activation='sigmoid' )(x)
 outputs = keras.layers.UpSampling2D(2**(depth-2))(x)
 model = keras.Model(inputs=inputs, outputs=outputs)
 return model<compute_test_metric>
%%time
train,n_col=create_lag_features(train,"total_cnt_for_items")
 train=train.fillna(0)
 train=train.drop(["total_cnt_for_items"],axis=1)
 numeric=numeric+n_col
def iou_loss(y_true, y_pred):
y_true = tf.reshape(y_true, [-1])
 y_pred = tf.reshape(y_pred, [-1])
 intersection = tf.reduce_sum(y_true * y_pred)
 score =(intersection + 1.) /(tf.reduce_sum(y_true)+ tf.reduce_sum(y_pred)- intersection + 1.)
return 1 - score
def iou_bce_loss(y_true, y_pred):
return 0.5 * keras.losses.binary_crossentropy(y_true, y_pred)+ 0.5 * iou_loss(y_true, y_pred)
 def mean_iou(y_true, y_pred):
y_pred = tf.round(y_pred)
 intersect = tf.reduce_sum(y_true * y_pred, axis=[1, 2, 3])
 union = tf.reduce_sum(y_true, axis=[1, 2, 3])+ tf.reduce_sum(y_pred, axis=[1, 2, 3])
 smooth = tf.ones(tf.shape(intersect))
return tf.reduce_mean(( intersect + smooth)/(union - intersect + smooth))
model = create_network(input_size=IMAGE_SIZE, channels=32, n_blocks=2, depth=4)
 model.compile(optimizer='adam',
loss=iou_bce_loss,
metrics=['accuracy', mean_iou])
 def cosine_annealing(x):
lr = 0.001
epochs = 20
return lr*(np.cos(np.pi*x/epochs)+1.) /2
learning_rate = tf.keras.callbacks.LearningRateScheduler(cosine_annealing)
 folder = '.. /input/stage_1_train_images'
train_gen = generator(folder, train_filenames, pneumonia_locations, batch_size=BATCH_SIZE, image_size=IMAGE_SIZE, shuffle=True, augment=True, predict=False)
 valid_gen = generator(folder, valid_filenames, pneumonia_locations, batch_size=BATCH_SIZE, image_size=IMAGE_SIZE, shuffle=False, predict=False)
 print(model.summary() )<train_model>
total_cnt_for_shops=sales_train_df[["date_block_num","shop_id","item_cnt_day"]].groupby(["date_block_num","shop_id"] ).sum()
total_cnt_for_shops.columns=["total_cnt_for_shops"]
total_cnt_for_shops.head()
history = model.fit_generator(train_gen, validation_data=valid_gen, callbacks=[learning_rate], epochs=15, shuffle=True )<define_variables>
sum_cnt_shops=total_cnt_for_shops.reset_index()
cnt_shops=sum_cnt_shops[["shop_id","total_cnt_for_shops"]].groupby("shop_id",as_index=False ).mean()
cnt_shops=cnt_shops.join(shops_df.set_index(["shop_id"]),on="shop_id",how="left")
 cnt_shops.head()
folder = '.. /input/stage_1_test_images'
test_filenames = os.listdir(folder)
 print('n test samples:', len(test_filenames))
test_gen = generator(folder, test_filenames, None, batch_size=16, image_size=IMAGE_SIZE, shuffle=False, predict=True)
 submission_dict = {}
for imgs, filenames in test_gen:
preds = model.predict(imgs)
 for pred, filename in zip(preds, filenames):
pred = resize(pred,(1024, 1024), mode='reflect')
 comp = pred[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
conf = np.mean(pred[y:y+height, x:x+width])
 predictionString += str(conf)+ ' ' + str(x)+ ' ' + str(y)+ ' ' + str(width)+ ' ' + str(height)+ ' '
filename = filename.split('.')[0]
submission_dict[filename] = predictionString
if len(submission_dict)>= len(test_filenames):
break
print("Done predicting...")
 sub = pd.DataFrame.from_dict(submission_dict,orient='index')
 sub.index.names = ['patientId']
sub.columns = ['PredictionString']
sub.to_csv('submission.csv' )<import_modules>
cnt_shops.loc[cnt_shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
cnt_shops['shop_city'] = cnt_shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 cnt_shops.loc[cnt_shops.shop_city == '!Якутск', 'shop_city'] = 'Якутск'
cnt_shops['shop_type'] = cnt_shops['shop_name'].str.split(' ' ).map(lambda x: x[1])
 cnt_shops.head()
import os
import csv
import random
import pydicom
import numpy as np
import pandas as pd
from skimage import measure
from skimage.transform import resize
import tensorflow as tf
from tensorflow import keras
from matplotlib import pyplot as plt<load_from_csv>
train=train.join(cnt_shops[["shop_id","shop_city","shop_type"]].set_index(["shop_id"]),on="shop_id",how="left")
 train.fillna(0,inplace=True)
 train["shop_city"]=train["shop_city"].astype(str)
 train["shop_type"]=train["shop_type"].astype(str)
 train.head().T
nodule_locations = {}
with open(os.path.join('.. /input/stage_1_train_labels.csv'), mode='r')as infile:
reader = csv.reader(infile)
 next(reader, None)
 for rows in reader:
filename = rows[0]
location = rows[1:5]
nodule = rows[5]
if nodule == '1':
location = [int(float(i)) for i in location]
if filename in nodule_locations:
nodule_locations[filename].append(location)
 else:
nodule_locations[filename] = [location]<define_variables>
categorical=categorical+["shop_city","shop_type"]
folder = '.. /input/stage_1_train_images'
filenames = os.listdir(folder)
 random.shuffle(filenames)
 n_valid_samples = 2560
train_filenames = filenames[n_valid_samples:]
valid_filenames = filenames[:n_valid_samples]
print('n train samples', len(train_filenames))
print('n valid samples', len(valid_filenames))
n_train_samples = len(filenames)- n_valid_samples<load_pretrained>
train=train.join(total_cnt_for_shops,on=["date_block_num","shop_id"],how="left")
 train.fillna(0,inplace=True)
 train["total_cnt_for_shops"]=train["total_cnt_for_shops"].astype(np.float16)
 train.head().T
class generator(keras.utils.Sequence):
def __init__(self, folder, filenames, nodule_locations=None, batch_size=32, image_size=512, shuffle=True, predict=False, augment = False):
self.folder = folder
self.filenames = filenames
self.nodule_locations = nodule_locations
self.batch_size = batch_size
self.image_size = image_size
self.augment = augment
self.shuffle = shuffle
self.predict = predict
self.on_epoch_end()
def __load__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
msk = np.zeros(img.shape)
 filename = filename.split('.')[0]
if filename in nodule_locations:
for location in nodule_locations[filename]:
x, y, w, h = location
msk[y:y+h, x:x+w] = 1
img = resize(img,(self.image_size, self.image_size), mode='reflect')
 msk = resize(msk,(self.image_size, self.image_size), mode='reflect')> 0.5
if self.augment and random.random() > 0.5:
img = np.fliplr(img)
 msk = np.fliplr(msk)
 img = np.expand_dims(img, -1)
 msk = np.expand_dims(msk, -1)
 return img, msk
def __loadpredict__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
img = resize(img,(self.image_size, self.image_size), mode='reflect')
 img = np.expand_dims(img, -1)
 return img
def __getitem__(self, index):
filenames = self.filenames[index*self.batch_size:(index+1)*self.batch_size]
if self.predict:
imgs = [self.__loadpredict__(filename)for filename in filenames]
imgs = np.array(imgs)
 return imgs, filenames
else:
items = [self.__load__(filename)for filename in filenames]
imgs, msks = zip(*items)
 imgs = np.array(imgs)
 msks = np.array(msks)
 return imgs, msks
def on_epoch_end(self):
if self.shuffle:
random.shuffle(self.filenames)
 def __len__(self):
if self.predict:
return int(np.ceil(len(self.filenames)/ self.batch_size))
else:
return int(len(self.filenames)/ self.batch_size )<define_search_model>
%%time
train,n_col=create_lag_features(train,"total_cnt_for_shops")
 train=train.fillna(0)
 train=train.drop(["total_cnt_for_shops"],axis=1)
 numeric=numeric+n_col
def create_downsample(channels, inputs):
x = keras.layers.BatchNormalization()(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 1, padding='same', use_bias=False )(x)
 x = keras.layers.MaxPool2D(2 )(x)
 return x
def create_resblock(channels, inputs):
x = keras.layers.BatchNormalization()(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 x = keras.layers.BatchNormalization()(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 return keras.layers.add([x, inputs])
 def create_network(input_size, channels, n_blocks=2, depth=5):
inputs = keras.Input(shape=(input_size, input_size, 1))
x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(inputs)
 for d in range(depth):
channels = channels * 2
x = create_downsample(channels, x)
 for b in range(n_blocks):
x = create_resblock(channels, x)
 x = keras.layers.BatchNormalization()(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(1, 1, activation='sigmoid' )(x)
 outputs = keras.layers.UpSampling2D(2**depth )(x)
 model = keras.Model(inputs=inputs, outputs=outputs)
 return model<compute_test_metric>
total_cnt_for_item_categories=sales_train_df[["date_block_num","item_category_id","item_cnt_day"]].groupby(["date_block_num","item_category_id"] ).sum()
total_cnt_for_item_categories.columns=["total_cnt_for_item_categories"]
total_cnt_for_item_categories.head()
def mean_iou(y_true, y_pred):
y_pred = tf.round(y_pred)
 intersect = tf.reduce_sum(y_true * y_pred, axis=[1, 2, 3])
 union = tf.reduce_sum(y_true, axis=[1, 2, 3])+ tf.reduce_sum(y_pred, axis=[1, 2, 3])
 smooth = tf.ones(tf.shape(intersect))
return tf.reduce_mean(( intersect + smooth)/(union - intersect + smooth))
model = create_network(input_size=256, channels=32, n_blocks=2, depth=4)
 model.compile(optimizer=keras.optimizers.Adam(lr=.01),loss=keras.losses.binary_crossentropy,metrics=['accuracy', mean_iou])
 folder = '.. /input/stage_1_train_images'
train_gen = generator(folder, train_filenames, nodule_locations, batch_size=32, image_size=256, shuffle=True, augment=True, predict=False)
 valid_gen = generator(folder, valid_filenames, nodule_locations, batch_size=32, image_size=256, shuffle=False, predict=False)
 history = model.fit_generator(train_gen, validation_data=valid_gen, epochs=3, shuffle=True, verbose=1 )<save_model>
sum_cnt_for_item_category["category"]=sum_cnt_for_item_category.item_category_name.str.split("-" ).str[0]
sum_cnt_for_item_category["subcategory"]=sum_cnt_for_item_category.item_category_name.str.split("-" ).str[1]
sum_cnt_for_item_category.head()
model.save("model.h5" )<predict_on_test>
train=train.join(items_df.set_index(["item_id"]),on=["item_id"],how="left")
 train=train.join(sum_cnt_for_item_category[["item_category_id","category","subcategory"]].set_index(["item_category_id"]),on=["item_category_id"],how="left")
 train.fillna(0,inplace=True)
 train["category"]=train["category"].astype(str)
 train["subcategory"]=train["subcategory"].astype(str)
 train.tail().T
folder = '.. /input/stage_1_test_images'
test_filenames = os.listdir(folder)
 print('n test samples:', len(test_filenames))
test_gen = generator(folder, test_filenames, None, batch_size=25, image_size=256, shuffle=False, predict=True)
 submission_dict = {}
for imgs, filenames in test_gen:
preds = model.predict(imgs)
 for pred, filename in zip(preds, filenames):
pred = resize(pred,(1024, 1024), mode='reflect')
 comp = pred[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
conf = np.mean(pred[y:y+height, x:x+width])
 predictionString += str(conf)+ ' ' + str(x)+ ' ' + str(y)+ ' ' + str(width)+ ' ' + str(height)+ ' '
filename = filename.split('.')[0]
submission_dict[filename] = predictionString
if len(submission_dict)>= len(test_filenames):
break
sub = pd.DataFrame.from_dict(submission_dict,orient='index')
 sub.index.names = ['patientId']
sub.columns = ['PredictionString']
sub.to_csv('submission1.csv' )<import_modules>
categorical=categorical+["category","subcategory"]
import os
import csv
import random
import pydicom
import numpy as np
import pandas as pd
from skimage import measure
from skimage.transform import resize
import tensorflow as tf
from tensorflow import keras
from matplotlib import pyplot as plt<load_from_csv>
train=train.join(total_cnt_for_item_categories,on=["date_block_num","item_category_id"],how="left")
 train.fillna(0,inplace=True)
 train["total_cnt_for_item_categories"]=train["total_cnt_for_item_categories"].astype(np.float16)
 train.head().T
pneumonia_locations = {}
with open(os.path.join('.. /input/stage_1_train_labels.csv'), mode='r')as infile:
reader = csv.reader(infile)
 next(reader, None)
 for rows in reader:
filename = rows[0]
location = rows[1:5]
pneumonia = rows[5]
if pneumonia == '1':
location = [int(float(i)) for i in location]
if filename in pneumonia_locations:
pneumonia_locations[filename].append(location)
 else:
pneumonia_locations[filename] = [location]<define_variables>
%%time
train,n_col=create_lag_features(train,"total_cnt_for_item_categories")
 train=train.fillna(0)
 train=train.drop(["total_cnt_for_item_categories"],axis=1)
 numeric=numeric+n_col
folder = '.. /input/stage_1_train_images'
filenames = os.listdir(folder)
 random.shuffle(filenames)
 n_valid_samples = 2560
train_filenames = filenames[n_valid_samples:]
valid_filenames = filenames[:n_valid_samples]
print('n train samples', len(train_filenames))
print('n valid samples', len(valid_filenames))
n_train_samples = len(filenames)- n_valid_samples<define_variables>
mean_price=sales_train_df[["date_block_num","shop_id","item_id","item_price"]].groupby(["date_block_num","shop_id","item_id"] ).mean()
mean_price.columns=["mean_price"]
mean_price.head()
class generator(keras.utils.Sequence):
def __init__(self, folder, filenames, pneumonia_locations=None, batch_size=32, image_size=256, shuffle=True, augment=False, predict=False):
self.folder = folder
self.filenames = filenames
self.pneumonia_locations = pneumonia_locations
self.batch_size = batch_size
self.image_size = image_size
self.shuffle = shuffle
self.augment = augment
self.predict = predict
self.on_epoch_end()
def __load__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
msk = np.zeros(img.shape)
 filename = filename.split('.')[0]
if filename in pneumonia_locations:
for location in pneumonia_locations[filename]:
x, y, w, h = location
msk[y:y+h, x:x+w] = 1
if self.augment and random.random() > 0.5:
img = np.fliplr(img)
 msk = np.fliplr(msk)
 img = resize(img,(self.image_size, self.image_size), mode='reflect')
 msk = resize(msk,(self.image_size, self.image_size), mode='reflect')> 0.5
img = np.expand_dims(img, -1)
 msk = np.expand_dims(msk, -1)
 return img, msk
def __loadpredict__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
img = resize(img,(self.image_size, self.image_size), mode='reflect')
 img = np.expand_dims(img, -1)
 return img
def __getitem__(self, index):
filenames = self.filenames[index*self.batch_size:(index+1)*self.batch_size]
if self.predict:
imgs = [self.__loadpredict__(filename)for filename in filenames]
imgs = np.array(imgs)
 return imgs, filenames
else:
items = [self.__load__(filename)for filename in filenames]
imgs, msks = zip(*items)
 imgs = np.array(imgs)
 msks = np.array(msks)
 return imgs, msks
def on_epoch_end(self):
if self.shuffle:
random.shuffle(self.filenames)
 def __len__(self):
if self.predict:
return int(np.ceil(len(self.filenames)/ self.batch_size))
else:
return int(len(self.filenames)/ self.batch_size )<define_search_model>
train=train.join(mean_price,on=["date_block_num","shop_id","item_id"],how="left")
 train.fillna(0,inplace=True)
 train["mean_price"]=train["mean_price"].astype(np.float16)
 train.head().T
def create_downsample(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 1, padding='same', use_bias=False )(x)
 x = keras.layers.MaxPool2D(2 )(x)
 return x
def create_resblock(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 return keras.layers.add([x, inputs])
 def create_network(input_size, channels, n_blocks=2, depth=4):
inputs = keras.Input(shape=(input_size, input_size, 1))
x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(inputs)
 for d in range(depth):
channels = channels * 2
x = create_downsample(channels, x)
 for b in range(n_blocks):
x = create_resblock(channels, x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(1, 1, activation='sigmoid' )(x)
 outputs = keras.layers.UpSampling2D(2**depth )(x)
 model = keras.Model(inputs=inputs, outputs=outputs)
 return model<compute_test_metric>
%%time
train,n_col=create_lag_features(train,"mean_price")
 train=train.fillna(0)
 train=train.drop(["mean_price"],axis=1)
 numeric=numeric+n_col
def iou_loss(y_true, y_pred):
y_true = tf.reshape(y_true, [-1])
 y_pred = tf.reshape(y_pred, [-1])
 intersection = tf.reduce_sum(y_true * y_pred)
 score =(intersection + 1.) /(tf.reduce_sum(y_true)+ tf.reduce_sum(y_pred)- intersection + 1.)
return 1 - score
def iou_bce_loss(y_true, y_pred):
return 0.5 * keras.losses.binary_crossentropy(y_true, y_pred)+ 0.5 * iou_loss(y_true, y_pred)
 def mean_iou(y_true, y_pred):
y_pred = tf.round(y_pred)
 intersect = tf.reduce_sum(y_true * y_pred, axis=[1, 2, 3])
 union = tf.reduce_sum(y_true, axis=[1, 2, 3])+ tf.reduce_sum(y_pred, axis=[1, 2, 3])
 smooth = tf.ones(tf.shape(intersect))
return tf.reduce_mean(( intersect + smooth)/(union - intersect + smooth))
model = create_network(input_size=256, channels=32, n_blocks=2, depth=4)
 model.compile(optimizer='adam',
loss=iou_bce_loss,
metrics=['accuracy', mean_iou])
 def cosine_annealing(x):
lr = 0.001
epochs = 20
return lr*(np.cos(np.pi*x/epochs)+1.) /2
learning_rate = tf.keras.callbacks.LearningRateScheduler(cosine_annealing)
 folder = '.. /input/stage_1_train_images'
train_gen = generator(folder, train_filenames, pneumonia_locations, batch_size=32, image_size=256, shuffle=True, augment=True, predict=False)
 valid_gen = generator(folder, valid_filenames, pneumonia_locations, batch_size=32, image_size=256, shuffle=False, predict=False)
 history = model.fit_generator(train_gen, validation_data=valid_gen, callbacks=[learning_rate], epochs=20, shuffle=True )<predict_on_test>
mean_price_for_items=sales_train_df[["date_block_num","item_id","item_price"]].groupby(["date_block_num","item_id"] ).mean()
mean_price_for_items.columns=["mean_price_for_items"]
mean_price_for_items.head()
folder = '.. /input/stage_1_test_images'
test_filenames = os.listdir(folder)
 print('n test samples:', len(test_filenames))
test_gen = generator(folder, test_filenames, None, batch_size=25, image_size=256, shuffle=False, predict=True)
 submission_dict = {}
for imgs, filenames in test_gen:
preds = model.predict(imgs)
 for pred, filename in zip(preds, filenames):
pred = resize(pred,(1024, 1024), mode='reflect')
 comp = pred[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
conf = np.mean(pred[y:y+height, x:x+width])
 predictionString += str(conf)+ ' ' + str(x)+ ' ' + str(y)+ ' ' + str(width)+ ' ' + str(height)+ ' '
filename = filename.split('.')[0]
submission_dict[filename] = predictionString
if len(submission_dict)>= len(test_filenames):
break
sub = pd.DataFrame.from_dict(submission_dict,orient='index')
 sub.index.names = ['patientId']
sub.columns = ['PredictionString']
sub.to_csv('submission.csv' )<import_modules>
train=train.join(mean_price_for_items,on=["date_block_num","item_id"],how="left")
 train.fillna(0,inplace=True)
 train["mean_price_for_items"]=train["mean_price_for_items"].astype(np.float16)
 train.head().T
import os
import csv
import random
import pydicom
import numpy as np
import pandas as pd
from skimage import io
from skimage import measure
from skimage.transform import resize
import tensorflow as tf
from tensorflow import keras
from matplotlib import pyplot as plt
import matplotlib.patches as patches<load_from_csv>
%%time
train,n_col=create_lag_features(train,"mean_price_for_items")
 train=train.fillna(0)
 train=train.drop(["mean_price_for_items"],axis=1)
 numeric=numeric+n_col
pneumonia_locations = {}
with open(os.path.join('.. /input/stage_1_train_labels.csv'), mode='r')as infile:
reader = csv.reader(infile)
 next(reader, None)
 for rows in reader:
filename = rows[0]
location = rows[1:5]
pneumonia = rows[5]
if pneumonia == '1':
location = [int(float(i)) for i in location]
if filename in pneumonia_locations:
pneumonia_locations[filename].append(location)
 else:
pneumonia_locations[filename] = [location]<define_variables>
mean_price_for_item_categories=sales_train_df[["date_block_num","item_category_id","item_price"]].groupby(["date_block_num","item_category_id"] ).mean()
mean_price_for_item_categories.columns=["mean_price_for_item_categories"]
mean_price_for_item_categories.head()
folder = '.. /input/stage_1_train_images'
filenames = os.listdir(folder)
 random.shuffle(filenames)
 n_valid_samples = 2560
train_filenames = filenames[n_valid_samples:]
valid_filenames = filenames[:n_valid_samples]
print('n train samples', len(train_filenames))
print('n valid samples', len(valid_filenames))
n_train_samples = len(filenames)- n_valid_samples<define_variables>
train=train.join(mean_price_for_item_categories,on=["date_block_num","item_category_id"],how="left")
 train.fillna(0,inplace=True)
 train["mean_price_for_item_categories"]=train["mean_price_for_item_categories"].astype(np.float16)
 train.head().T
class generator(keras.utils.Sequence):
def __init__(self, folder, filenames, pneumonia_locations=None, batch_size=32, image_size=256, shuffle=True, augment=False, predict=False):
self.folder = folder
self.filenames = filenames
self.pneumonia_locations = pneumonia_locations
self.batch_size = batch_size
self.image_size = image_size
self.shuffle = shuffle
self.augment = augment
self.predict = predict
self.on_epoch_end()
def __load__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
msk = np.zeros(img.shape)
 filename = filename.split('.')[0]
if filename in self.pneumonia_locations:
for location in self.pneumonia_locations[filename]:
x, y, w, h = location
msk[y:y+h, x:x+w] = 1
img = resize(img,(self.image_size, self.image_size), mode='reflect')
 msk = resize(msk,(self.image_size, self.image_size), mode='reflect')> 0.5
if self.augment and random.random() > 0.5:
img = np.fliplr(img)
 msk = np.fliplr(msk)
 img = np.expand_dims(img, -1)
 msk = np.expand_dims(msk, -1)
 return img, msk
def __loadpredict__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
img = resize(img,(self.image_size, self.image_size), mode='reflect')
 img = np.expand_dims(img, -1)
 return img
def __getitem__(self, index):
filenames = self.filenames[index*self.batch_size:(index+1)*self.batch_size]
if self.predict:
imgs = [self.__loadpredict__(filename)for filename in filenames]
imgs = np.array(imgs)
 return imgs, filenames
else:
items = [self.__load__(filename)for filename in filenames]
imgs, msks = zip(*items)
 imgs = np.array(imgs)
 msks = np.array(msks)
 return imgs, msks
def on_epoch_end(self):
if self.shuffle:
random.shuffle(self.filenames)
 def __len__(self):
if self.predict:
return int(np.ceil(len(self.filenames)/ self.batch_size))
else:
return int(len(self.filenames)/ self.batch_size )<define_search_model>
%%time
train,n_col=create_lag_features(train,"mean_price_for_item_categories")
 train=train.fillna(0)
 train=train.drop(["mean_price_for_item_categories"],axis=1)
 numeric=numeric+n_col
def create_downsample(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 1, padding='same', use_bias=False )(x)
 x = keras.layers.MaxPool2D(2 )(x)
 return x
def create_resblock(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 return keras.layers.add([x, inputs])
 def create_network(input_size, channels, n_blocks=2, depth=4):
inputs = keras.Input(shape=(input_size, input_size, 1))
x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(inputs)
 for d in range(depth):
channels = channels * 2
x = create_downsample(channels, x)
 for b in range(n_blocks):
x = create_resblock(channels, x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(1, 1, activation='sigmoid' )(x)
 outputs = keras.layers.UpSampling2D(2**depth )(x)
 model = keras.Model(inputs=inputs, outputs=outputs)
 return model<compute_test_metric>
sales_train_df["revenue"]=sales_train_df["item_price"]*sales_train_df["item_cnt_day"]
total_revenue=sales_train_df[["date_block_num","shop_id","revenue"]].groupby(["date_block_num","shop_id"] ).sum()
total_revenue.columns=["total_shop_revenue"]
total_revenue.head()
def iou_loss(y_true, y_pred):
y_true = tf.reshape(y_true, [-1])
 y_pred = tf.reshape(y_pred, [-1])
 intersection = tf.reduce_sum(y_true * y_pred)
 score =(intersection + 1.) /(tf.reduce_sum(y_true)+ tf.reduce_sum(y_pred)- intersection + 1.)
return 1 - score
def iou_bce_loss(y_true, y_pred):
return 0.5 * keras.losses.binary_crossentropy(y_true, y_pred)+ 0.5 * iou_loss(y_true, y_pred)
 def mean_iou(y_true, y_pred):
y_pred = tf.round(y_pred)
 intersect = tf.reduce_sum(y_true * y_pred, axis=[1, 2, 3])
 union = tf.reduce_sum(y_true, axis=[1, 2, 3])+ tf.reduce_sum(y_pred, axis=[1, 2, 3])
 smooth = tf.ones(tf.shape(intersect))
return tf.reduce_mean(( intersect + smooth)/(union - intersect + smooth))
model = create_network(input_size=256, channels=32, n_blocks=2, depth=4)
 model.compile(optimizer='adam',
loss=iou_bce_loss,
metrics=['accuracy', mean_iou])
 def cosine_annealing(x):
lr = 0.001
epochs = 25
return lr*(np.cos(np.pi*x/epochs)+1.) /2
learning_rate = tf.keras.callbacks.LearningRateScheduler(cosine_annealing)
 folder = '.. /input/stage_1_train_images'
train_gen = generator(folder, train_filenames, pneumonia_locations, batch_size=32, image_size=256, shuffle=True, augment=True, predict=False)
 valid_gen = generator(folder, valid_filenames, pneumonia_locations, batch_size=32, image_size=256, shuffle=False, predict=False)
 history = model.fit_generator(train_gen, validation_data=valid_gen, callbacks=[learning_rate], epochs=25, workers=4, use_multiprocessing=True )<predict_on_test>
train=train.join(total_revenue,on=["date_block_num","shop_id"],how="left")
 train["total_shop_revenue"]=train["total_shop_revenue"].astype(np.float32)
 train.fillna(0,inplace=True)
 train.head().T
for imgs, msks in valid_gen:
preds = model.predict(imgs)
 f, axarr = plt.subplots(4, 8, figsize=(20,15))
axarr = axarr.ravel()
axidx = 0
for img, msk, pred in zip(imgs, msks, preds):
axarr[axidx].imshow(img[:, :, 0])
 comp = msk[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
axarr[axidx].add_patch(patches.Rectangle(( x,y),width,height,linewidth=2,edgecolor='b',facecolor='none'))
comp = pred[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
axarr[axidx].add_patch(patches.Rectangle(( x,y),width,height,linewidth=2,edgecolor='r',facecolor='none'))
axidx += 1
plt.show()
break<predict_on_test>
%%time
train,n_col=create_lag_features(train,"total_shop_revenue")
 train=train.fillna(0.0)
 train=train.drop(["total_shop_revenue"],axis=1)
 numeric=numeric+n_col
folder = '.. /input/stage_1_test_images'
test_filenames = os.listdir(folder)
 print('n test samples:', len(test_filenames))
test_gen = generator(folder, test_filenames, None, batch_size=25, image_size=256, shuffle=False, predict=True)
 submission_dict = {}
for imgs, filenames in test_gen:
preds = model.predict(imgs)
 for pred, filename in zip(preds, filenames):
pred = resize(pred,(1024, 1024), mode='reflect')
 comp = pred[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
conf = np.mean(pred[y:y+height, x:x+width])
 predictionString += str(conf)+ ' ' + str(x)+ ' ' + str(y)+ ' ' + str(width)+ ' ' + str(height)+ ' '
filename = filename.split('.')[0]
submission_dict[filename] = predictionString
if len(submission_dict)>= len(test_filenames):
break
sub = pd.DataFrame.from_dict(submission_dict,orient='index')
 sub.index.names = ['patientId']
sub.columns = ['PredictionString']
sub.to_csv('submission.csv' )<load_from_csv>
numeric=numeric+["date_block_num","item_cnt_day"]
train_df1=pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test_df1=pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
<concatenate>
X_train_categorical=train[categorical]
X_test_categorical=test_df[categorical]
X_train_categorical["subcategory"]=X_train_categorical["subcategory"].astype(str)
 X_test_categorical["subcategory"]=X_test_categorical["subcategory"].astype(str)
 X_test_categorical["year"]=X_test_categorical["year"].astype(np.int16)
 X_test_categorical["month"]=X_test_categorical["month"].astype(np.int8)
 X_test_categorical.loc[X_test_categorical.category=="PC ","category"]="Игры PC "
X_test_categorical.loc[X_test_categorical.subcategory==" Гарнитуры/Наушники","subcategory"]=" Аксессуары для игр"
df1=pd.concat([train_df1,test_df1],axis=0 )<create_dataframe>
for feature in categorical:
le=LabelEncoder()
print(feature)
 X_train_categorical[feature]=le.fit_transform(X_train_categorical[feature])
 X_test_categorical[feature]=le.transform(X_test_categorical[feature] )
Missing_df=pd.DataFrame(df1.isnull().sum().sort_values(ascending=False)/len(df1))
Missing_df.head(40 )<drop_column>
X_train_numeric=train[numeric]
X_test_numeric=test_df[numeric]
df1.drop(['PoolQC','MiscFeature','Alley','Fence'],axis=1,inplace=True )<data_type_conversions>
label_cat_not_num_train=pd.concat([X_train_categorical,X_train_numeric],axis=1)
 label_cat_not_num_train.head().T
for feature in cat_features_list:
df1[feature].fillna(df1[feature].mode() [0],inplace=True)
<data_type_conversions>
label_cat_not_num_test=pd.concat([X_test_categorical,X_test_numeric],axis=1)
 label_cat_not_num_test.head().T
num_features_list.remove('SalePrice')
 for feature in num_features_list:
df1[feature].fillna(df1[feature].median() ,inplace=True )<sort_values>
def downcast_type(df):
for feature in categorical:
df[feature]=df[feature].astype(np.int8 )
df1.isnull().sum().sort_values(ascending=False ).head(50 )<feature_engineering>
X_train=label_cat_not_num_train[label_cat_not_num_train.date_block_num<=19]
X_val=label_cat_not_num_train[label_cat_not_num_train.date_block_num>19]
X_train=X_train.drop(["date_block_num"],axis=1)
 X_val=X_val.drop(["date_block_num"],axis=1 )
df1['YearsOld']=df1['YrSold']-df1['YearBuilt']<sort_values>
y_train=X_train["item_cnt_day"]
y_val=X_val["item_cnt_day"]
X_val=X_val.drop(["item_cnt_day"],axis=1)
 X_train=X_train.drop(["item_cnt_day"],axis=1 )
df1.skew().sort_values(ascending=False )<drop_column>
def RMSE(y,predictions):
return np.sqrt(mean_squared_error(y,predictions))
scorer=make_scorer(RMSE,False )
skewed_fearures=df1.skew().sort_values(ascending=False ).head(20 ).index.tolist()
skewed_fearures.remove('SalePrice')
 df1[skewed_fearures]=np.log1p(df1[skewed_fearures] )<categorify>
rf = RandomForestRegressor(n_estimators=50, max_depth=16,max_features=0.4,min_samples_leaf=10,n_jobs=-1)
 rf.fit(X_train, y_train )
df2=pd.get_dummies(df1[cat_features_list],drop_first=True)
 df1.drop(cat_features_list,axis=1,inplace=True)
<count_duplicates>
predictions = []
for tree in rf.estimators_:
predictions.append(tree.predict(X_val))
df3=pd.concat([df1,df2],axis=1)
 d3 = df3.loc[:,~df3.columns.duplicated() ]<drop_column>
predictions = np.vstack(predictions )
df3.drop(['Id'],axis=1,inplace=True)
 df3.shape<prepare_x_and_y>
cum_mean = np.cumsum(predictions, axis=0)/np.arange(1, predictions.shape[0] + 1)[:, None]
X=df3.drop('SalePrice',axis=1)
 Y=df3['SalePrice']
X_train=X.iloc[:1460]
X_test=X.iloc[1460:,]
Y_train=Y.iloc[:1460]
Y_test=Y.iloc[1460:]
Y_train=np.log1p(Y_train )<train_model>
scores = []
for pred in cum_mean:
scores.append(np.sqrt(mean_squared_error(y_val,pred)) )
lr_model=LinearRegression()
lr_model.fit(X_train,Y_train)
 Y_pred_lr=lr_model.predict(X_test)
 Y_lr_train=lr_model.predict(X_train )<compute_test_metric>
pred=rf.predict(X_val )
MSE=np.sqrt(mean_squared_error(Y_train,Y_lr_train))
print("RMSE of the Linear Regression Model is :",np.sqrt(MSE)) 
<compute_test_metric>
score=np.sqrt(mean_squared_error(y_val,pred))
score
r2_score=lr_model.score(X_train,Y_train)
 print("R2_score of Liner Regression",r2_score )<compute_test_metric>
y_train=label_cat_not_num_train["item_cnt_day"]
label_cat_not_num_train=label_cat_not_num_train.drop(["item_cnt_day"],axis=1)
 label_cat_not_num_train=label_cat_not_num_train.drop(["date_block_num"],axis=1)
 label_cat_not_num_test=label_cat_not_num_test.drop(["item_cnt_day"],axis=1)
 label_cat_not_num_test=label_cat_not_num_test.drop(["date_block_num"],axis=1 )
adj_r2 =(1 -(1 - r2_score)*(( X_train.shape[0] - 1)/ 
(X_train.shape[0] - X_train.shape[1] - 1)))
 print("Adjusted R_Sqaured of Linear Regression Model is :",adj_r2 )<choose_model_class>
rf = RandomForestRegressor(n_estimators=50, max_depth=16,max_features=0.4,min_samples_leaf=10,n_jobs=-1)
 rf.fit(label_cat_not_num_train,y_train )
classifier=xgboost.XGBRegressor()
booster=['gbtree','gblinear']
base_score=[0.25,0.5,0.75,1]<define_search_space>
predictions=rf.predict(label_cat_not_num_test )
n_estimators = [100, 500, 900, 1100, 1500]
max_depth = [2, 3, 5, 10, 15]
booster=['gbtree','gblinear']
learning_rate=[0.05,0.1,0.15,0.20]
min_child_weight=[1,2,3,4]
hyperparameter_grid = {
'n_estimators': n_estimators,
'max_depth':max_depth,
'learning_rate':learning_rate,
'min_child_weight':min_child_weight,
'booster':booster,
'base_score':base_score
}<choose_model_class>
sample_submissions["item_cnt_month"]=predictions
sample_submissions.head()
<train_model><EOS> 
sample_submissions.to_csv("random_forest_lagged_features_6.csv",index=False )