Kaggle competition: Outbrain Click Prediction
with open(".. /input/intermediatetgs/test_index.obj", "rb")as f:
indexes = _pickle.load(f )<compute_test_metric>
def OHEncode(df, cols):
for x in cols:
x_ohe = pd.get_dummies(df[x], prefix=x)
 df = pd.concat([df, x_ohe], axis=1)
 df = df.drop([x], axis=1)
 return df
t1 = time.time()
pred_dict = {idx: rle_encode(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(indexes)) }
t2 = time.time()
print(f"Usedtime = {t2-t1} s" )<save_to_csv>
cols = ['platform']
sub = pd.DataFrame.from_dict(pred_dict,orient="index")
 sub.index.names = ["id"]
sub.columns = ["rle_mask"]
sub.to_csv("submission4.csv" )<train_model>
testing=False
chunksize=50000
clf = LogisticRegression()
<set_options>
train = pd.read_csv(".. /input/train-featured/clicks_events_full.csv", iterator=True,chunksize=chunksize )
plt.style.use('seaborn-white')
 sns.set_style("white")
 t_start = time.time()<train_model>
print('Chunks training')
 for chunk in train:
chunk = chunk.drop(['geo_location','dayofweek','hour'], axis=1)
 chunk = OHEncode(chunk,cols)
 predictors=[x for x in chunk.columns if x not in ['display_id','clicked']]
chunk=chunk.fillna(0.0)
 clf.fit(chunk[predictors], chunk["clicked"])
 if testing:
break
train=''
img_size_ori = 101
img_size_target = 128
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
<load_from_csv>
test = pd.read_csv(".. /input/test-featured/test_events_full.csv",iterator=True,chunksize=chunksize )
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 print(train_df.shape)
 print(train_df.head)
 train_df = train_df.join(depths_df)
 print(train_df.shape)
 print(train_df.head)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
print("test_df")
 print(test_df.head )<feature_engineering>
print('Testing')
 predY=[]
for chunk in test:
init_chunk_size=len(chunk)
 chunk = chunk.drop(['geo_location','dayofweek','hour'], axis=1)
 chunk = OHEncode(chunk,cols)
 chunk=chunk.fillna(0.0)
 chunk_pred=list(clf.predict_proba(chunk[predictors] ).astype(float)[:,1])
 predY += chunk_pred
if testing:
break
print('Done Testing' )
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx),grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
print('Preparing for Submission')
 test=''
test= pd.read_csv('.. /input/outbrain-click-prediction/clicks_test.csv' )
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
results=pd.concat(( test,pd.DataFrame(predY)) ,axis=1,ignore_index=True)
 print(results.head(10))
train_df["coverage"] = train_df.masks.map(np.sum)/ 3/pow(img_size_ori, 2 )<categorify>
results.columns = ['display_id','ad_id','clicked']
print(results.head(10))
def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )<split>
results = results.sort_values(by=['display_id','clicked'], ascending=[True, False])
 results = results.reset_index(drop=True )
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class )<define_search_model>
results2 = results.copy()
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation == True:
x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate = False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate:
x = BatchActivate(x)
 return x 
<choose_model_class>
submission_data = results2.groupby('display_id' ).ad_id.apply(lambda x: " ".join(map(str,x)) ).reset_index()
def build_model(input_layer, start_neurons):
conv1 = Conv2D(start_neurons * 1 ,(3,3), activation=None, padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1,True)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(0.25 )(pool1)
 conv2 = Conv2D(start_neurons * 2 ,(3,3), activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2,True)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(0.5 )(pool2)
 conv3 = Conv2D(start_neurons * 4 ,(3,3), activation="relu", padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4,True)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(0.5 )(pool3)
 conv4 = Conv2D(start_neurons * 8 ,(3,3), activation="relu", padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8,True)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(0.5 )(pool4)
 convm = Conv2D(start_neurons * 16 ,(3,3), activation="relu", padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16,True)
 deconv4 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(0.5 )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3,3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8, True)
 deconv3 = Conv2DTranspose(start_neurons * 4 ,(3, 3), strides=(2, 2), padding="same" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(0.5 )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3,3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4, True)
 deconv2 = Conv2DTranspose(start_neurons * 2 ,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(0.5 )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3,3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2, True)
 deconv1 = Conv2DTranspose(start_neurons * 1 ,(3, 3), strides=(2, 2), padding="same" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(0.5 )(uconv1)
 uconv1 = Conv2D(start_neurons * 1 *4,(3, 3), activation="relu", padding="same" )(uconv1)
 uconv1 = Conv2D(start_neurons * 1 *4,(3, 3), activation="relu", padding="same" )(uconv1)
 output_layer = Conv2D(1,(1,1), padding="same", activation="sigmoid" )(uconv1)
 return output_layer
input_layer = Input(( img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 32 )<choose_model_class>
submission_data.to_csv('submission_feature_removed.csv', index=False )