Kaggle competition: Google Analytics Customer Revenue Prediction
datagen_try = ImageDataGenerator(rotation_range=15,
width_shift_range = 0.25,
height_shift_range = 0.25,
shear_range = 25,
zoom_range = 0.4,)
 datagen_try.fit(x_train)
 for x_batch, y_batch in datagen_try.flow(x_train, y_train, batch_size=9):
	
	for i in range(0, 9):
		plt.subplot(330 + 1 + i)
 		plt.imshow(x_batch[i].reshape(28, 28), cmap=plt.get_cmap('gray'))
	
	plt.show()
	break<define_search_model>
warnings.simplefilter('error', SettingWithCopyWarning)
 gc.enable()
%matplotlib inline
datagen_train = ImageDataGenerator(rotation_range = 10,
width_shift_range = 0.25,
height_shift_range = 0.25,
shear_range = 10,
zoom_range = 0.1,
horizontal_flip = False)
 datagen_val = ImageDataGenerator()
step_train = x_train.shape[0] // batch_size
step_val = x_val.shape[0] // batch_size
learning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau(
 monitor='loss',
factor=0.25,
patience=2,
verbose=1,
mode="auto",
min_delta=0.0001,
cooldown=0,
min_lr=0.00001 
)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=300, restore_best_weights=True)
<train_model>
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 train.shape, test.shape
history = model.fit_generator(datagen_train.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=len(x_train)//batch_size,
epochs=epochs,
validation_data=(x_val, y_val),
validation_steps=50,
callbacks=[learning_rate_reduction, es],
verbose=2 )<train_model>
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']
tf.keras.utils.plot_model(model, to_file="model.png", show_shapes=True )<compute_test_metric>
train['target'] = y_reg
for df in [train, test]:
df['vis_date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['vis_date'].dt.dayofweek
df['sess_date_hours'] = df['vis_date'].dt.hour
df['sess_date_dom'] = df['vis_date'].dt.day
df.sort_values(['fullVisitorId', 'vis_date'], ascending=True, inplace=True)
 df['next_session_1'] =(
 df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['next_session_2'] =(
 df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(-1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['nb_pageviews'] = df['date'].map(
 df[['date', 'totals.pageviews']].groupby('date')['totals.pageviews'].sum() 
)
 df['ratio_pageviews'] = df['totals.pageviews'] / df['nb_pageviews']
y_reg = train['target']
del train['target']
model.evaluate(x_val, y_val, verbose=2);<predict_on_test>
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime', 'vis_date', 'nb_sessions', 'max_visits'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]
y_predicted = model.predict(x_val)
 y_grand_truth = y_val
y_predicted = np.argmax(y_predicted,axis=1)
 y_grand_truth = np.argmax(y_grand_truth,axis=1)
 cm = confusion_matrix(y_grand_truth, y_predicted )<prepare_output>
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )
scores = np.zeros(( 10,3))
def calc_F1(num):
TP = cm[num,num]
FN = np.sum(cm[num,:])-cm[num,num]
FP = np.sum(cm[:,num])-cm[num,num]
precision = TP/(TP+FP)
 recall = TP/(TP+FN)
 F1_score = 2*(recall * precision)/(recall + precision)
 return precision, recall, F1_score
for i in range(10):
precision, recall, F1_score = calc_F1(i)
 scores[i,:] = precision, recall, F1_score
scores_frame = pd.DataFrame(scores,columns=["Precision", "Recall", "F1 Score"], index=[list(range(0, 10)) ] )<load_from_csv>
folds = get_folds(df=train, n_splits=5)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
print(train_features)
 importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=100,
eval_metric='rmse' 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5
raw_dig = pd.read_csv(".. /input/Kannada-MNIST/Dig-MNIST.csv")
 raw_dig.head()
x_dig = raw_dig.iloc[:, 1:].values.astype('float32')/ 255
y_dig = raw_dig.iloc[:, 0].values
x_dig = x_dig.reshape(-1,28,28,1)
 y_dig = to_categorical(y_dig)
 model.evaluate(x_dig, y_dig, verbose=2 )<load_from_csv>
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds
sample_sub=pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 raw_test_id=raw_test.id
raw_test=raw_test.drop("id",axis="columns")
 raw_test=raw_test / 255
test=raw_test.values.reshape(-1,28,28,1)
 test.shape<save_to_csv>
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub=model.predict(test)
 sub=np.argmax(sub,axis=1)
 sample_sub['label']=sub
sample_sub.to_csv('submission.csv',index=False )<install_modules>
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )
!cp '.. /input/weightedboxesfusion'.-r
!pip install --no-deps './weightedboxesfusion' > /dev/null<import_modules>
trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape
np.random.seed(123)
 torch.manual_seed(123 )<define_variables>
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )
N_CLUSTERS = 1
PL_ROUNDS = 5
PL_EPOCHS = 5
detection_threshold = 0.5
image_size = 1024<load_from_csv>
sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape
dataset_path = '/kaggle/input/global-wheat-detection'
original_train_path = '/kaggle/input/original-train/original_train'
weights_path = '/kaggle/input/manual-weights-7/e152.pth'
train_path = join(dataset_path, 'train')
 test_path = join(dataset_path, 'test')
 train_df = pd.read_csv(join(original_train_path, 'train.csv'))
train_df['image_path'] = [join(original_train_path, s, i + '.jpeg')for i, s in train_df[['image_id', 'source']].values]
test_df = pd.read_csv(join(dataset_path, 'sample_submission.csv'))<feature_engineering>
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()
id_to_path = {}
for idx, row in train_df.iterrows() :
image_id = row['image_id']
if image_id in id_to_path.keys() : continue
id_to_path[image_id] = row['image_path']
for idx, row in test_df.iterrows() :
image_id = row['image_id']
if image_id in id_to_path.keys() : continue
id_to_path[image_id] = join(test_path, image_id + '.jpg' )<prepare_output>
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds[oof_preds < 0] = 0
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_preds += _preds / len(folds)
 mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5
class WheatTrainDataset(Dataset):
def __init__(self, dataframe, transforms=None, test=False):
super().__init__()
self.df = dataframe
self.image_ids = pd.unique(dataframe['image_id'])
 self.image_paths = [id_to_path[image_id] for image_id in self.image_ids]
self.length = len(self.image_ids)
 self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image_path = self.image_paths[index]
image = self.load_image(image_path)
 boxes = self.load_boxes(image_id)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {'boxes': boxes, 'labels': labels, 'image_id': torch.tensor([index])}
if self.transforms:
sample = {'image': image, 'bboxes': target['boxes'], 'labels': labels}
sample = self.transforms(**sample)
 image, boxes = sample['image'], sample['bboxes']
boxes = self.filter_boxes(boxes)
 if len(boxes):
target['boxes'] = torch.stack([torch.tensor(box, dtype=torch.float32)for box in zip(*boxes)] ).permute(1, 0)
 else:
return self.__getitem__(np.random.randint(self.length))
return image, target, image_id
def __len__(self)-> int:
return self.length
def load_image(self, image_path):
image = cv2.imread(image_path , cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
return image
def load_boxes(self, image_id):
records = self.df[self.df['image_id'] == image_id]
if 'x1' in records.columns and 'y1' in records.columns:
return records[['x', 'y', 'x1', 'y1']].values
else:
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return boxes
def filter_boxes(self, boxes):
min_length = 13
min_area = 400
max_area = 145360
max_length_ratio = 18
min_length_ratio = 1.0/max_length_ratio
boxes_out = []
for box in boxes:
x, y, x1, y1 = box
w = x1 - x
if w < min_length: continue
h = y1 - y
if h < min_length: continue
area = w * h
if area < 400 or area > 145360: continue
length_ratio = w / h
if length_ratio < min_length_ratio or length_ratio > max_length_ratio: continue
boxes_out.append(box)
 return boxes_out
def get_sample_weights(self):
weights = []
for image_id in self.image_ids:
w, h, source = self.df[self.df['image_id'] == image_id][['width', 'height', 'source']].values[0]
if w == 3072:
weight = 8
elif h == 2048:
weight = 5
elif w == 2048:
weight = 2.5
else:
weight = 1
if source == 'test': weight *= 10
weights.append(weight)
 return np.array(weights)
 class WheatTestDataset(Dataset):
def __init__(self, dataframe, image_dir, image_size=1024, onfly=False, tta=True):
super().__init__()
self.image_ids = dataframe['image_id'].unique()
self.df = dataframe
self.image_dir = image_dir
self.image_size = image_size
self.onfly = onfly
self.tta = tta
def load_image(self, image_id):
image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 image = cv2.resize(image,(self.image_size, self.image_size))
image = image.astype(np.float32)
 image /= 255.0
return image
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = self.load_image(image_id)
 if self.tta:
all_images = apply_tta(image)
 return all_images, image_id
else:
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def collate_fn(batch): return tuple(zip(*batch))<compute_train_metric>
sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('new_test.csv', index=True )