Kaggle competition: PetFinder.my Adoption Prediction
class Landmark2020_DataGenerator(tf.keras.utils.Sequence):
def __init__(self, paths, labels, batch_size, shape, shuffle = False, use_cache = False, augment = False):
self.paths, self.labels = paths, labels
self.batch_size = batch_size
self.shape = shape
self.shuffle = shuffle
self.use_cache = use_cache
self.augment = augment
if use_cache == True:
self.cache = np.zeros(( paths.shape[0], shape[0], shape[1], shape[2]), dtype=np.float16)
 self.is_cached = np.zeros(( paths.shape[0]))
self.on_epoch_end()
def __len__(self):
return int(np.ceil(len(self.paths)/ float(self.batch_size)))
 def __getitem__(self, idx):
indexes = self.indexes[idx * self.batch_size :(idx+1)* self.batch_size]
paths = self.paths[indexes]
X = np.zeros(( paths.shape[0], self.shape[0], self.shape[1], self.shape[2]))
if self.use_cache == True:
X = self.cache[indexes]
for i, path in enumerate(paths[np.where(self.is_cached[indexes] == 0)]):
image = self.__load_image(path)
 self.is_cached[indexes[i]] = 1
self.cache[indexes[i]] = image
X[i] = image
else:
for i, path in enumerate(paths):
X[i] = self.__load_image(path)
 y = self.labels[indexes]
if self.augment == True:
seq = iaa.Sequential([
iaa.OneOf([
iaa.Fliplr(0.5),
iaa.ContrastNormalization(( 0.75, 1.5)) ,
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5),
iaa.Multiply(( 0.8, 1.2), per_channel=0.2),
iaa.Affine(rotate=0),
iaa.Fliplr(0.5),
])], random_order=True)
 X = np.concatenate(( X, seq.augment_images(X), seq.augment_images(X), seq.augment_images(X)) , 0)
 y = np.concatenate(( y, y, y, y), 0)
 return X, y
def on_epoch_end(self):
self.indexes = np.arange(len(self.paths))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __iter__(self):

 for item in(self[i] for i in range(len(self))):
yield item
def __load_image(self, path):
image_norm = cv2_imread(path)/255.0
im = cv2_resize(image_norm,(IMAGE_SIZE,IMAGE_SIZE))
return im<count_unique_values>
df_train = pd.read_csv('.. /input/train/train.csv')
 df_breed = pd.read_csv('.. /input/breed_labels.csv')
 df_color = pd.read_csv('.. /input/color_labels.csv')
 df_state = pd.read_csv('.. /input/state_labels.csv')
 df_test = pd.read_csv('.. /input/test/test.csv' )
nlabls = topclass_train['landmark_id'].nunique()<import_modules>
df_train.Name = df_train.Name.fillna('')
 df_test.Name = df_test.Name.fillna('' )
from tensorflow.keras.applications import InceptionV3<define_variables>
df_train['Name'] = df_train['Name'].replace('No Name Yet', '')
 df_test['Name'] = df_test['Name'].replace('No Name Yet', '' )
NN_branches = 20
DENSE_UNITS = 2048<choose_model_class>
df_train['name_len'] = df_train.Name.str.len()
df_test['name_len'] = df_test.Name.str.len()
base_model = InceptionV3(input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3),
weights=None, include_top=False)
 base_model.load_weights(".. /input/keraspretrainedmodel/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5")
 input_image = Input(( IMAGE_SIZE, IMAGE_SIZE, 3), dtype = tf.uint8)
 x = tf.cast(input_image, tf.float32)
 x = tf.keras.applications.inception_v3.preprocess_input(x)
 x = base_model(input_image)
 x = GlobalMaxPooling2D()(x)
 x = Flatten()(x)
 x = Dense(DENSE_UNITS,activation='relu' )(x)
 x = tf.keras.layers.LayerNormalization()(x)
 embedding_model = Model(inputs = input_image,outputs = x,name='embedding_model')
 cat_out_layers = []
output = None
for i in range(1,NN_branches+1):
cat = Dense(DENSE_UNITS//NN_branches,activation = 'relu',name = 'cat'+str(i))(embedding_model.output)
 if i != NN_branches:
cat_out = Dense(nlabls//NN_branches, activation='softmax', name = 'cat'+str(i)+'_out', kernel_initializer = glorot_uniform(seed=0))(cat)
 else:
cat_out = Dense(nlabls//NN_branches+nlabls%NN_branches, activation='softmax', name = 'cat'+str(i)+'_out', kernel_initializer = glorot_uniform(seed=0))(cat)
 cat_out_layers.append(cat_out)
 output = Concatenate()(cat_out_layers)
 model = Model(inputs=[input_image], outputs=[output])
 model.summary()<compute_test_metric>
import json
def top_5_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=5 )<set_options>
train_sentiment_path = '.. /input/train_sentiment/'
test_sentiment_path = '.. /input/test_sentiment/'
train_meta_path = '.. /input/train_metadata/'
test_meta_path = '.. /input/test_metadata/'
for layer in base_model.layers:
layer.trainable = False<choose_model_class>
df_train[['desc_magnitude', 'desc_score']] = df_train['PetID'].progress_apply(lambda x: get_sentiment(x, train_sentiment_path))
df_test[['desc_magnitude', 'desc_score']] = df_test['PetID'].progress_apply(lambda x: get_sentiment(x, test_sentiment_path))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc',top_5_accuracy])
 model.summary()<set_options>
target = df_train['AdoptionSpeed']
train_id = df_train['PetID']
test_id = df_test['PetID']
df_train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 df_test.drop(['PetID'], axis=1, inplace=True )
gc.collect()<find_best_params>
%%time
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 df_train.loc[:, 'vertex_x'] = vertex_xs
df_train.loc[:, 'vertex_y'] = vertex_ys
df_train.loc[:, 'bounding_confidence'] = bounding_confidences
df_train.loc[:, 'bounding_importance'] = bounding_importance_fracs
df_train.loc[:, 'dominant_blue'] = dominant_blues
df_train.loc[:, 'dominant_green'] = dominant_greens
df_train.loc[:, 'dominant_red'] = dominant_reds
df_train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
df_train.loc[:, 'dominant_score'] = dominant_scores
df_train.loc[:, 'label_description'] = label_descriptions
df_train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 df_test.loc[:, 'vertex_x'] = vertex_xs
df_test.loc[:, 'vertex_y'] = vertex_ys
df_test.loc[:, 'bounding_confidence'] = bounding_confidences
df_test.loc[:, 'bounding_importance'] = bounding_importance_fracs
df_test.loc[:, 'dominant_blue'] = dominant_blues
df_test.loc[:, 'dominant_green'] = dominant_greens
df_test.loc[:, 'dominant_red'] = dominant_reds
df_test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
df_test.loc[:, 'dominant_score'] = dominant_scores
df_test.loc[:, 'label_description'] = label_descriptions
df_test.loc[:, 'label_score'] = label_scores
paths, labels,_ = getTrainParams()<set_options>
image_meta_col = ['vertex_x', 'vertex_y', 'bounding_confidence', 'bounding_importance', 'dominant_blue', 'dominant_green', 'dominant_red', 'dominant_pixel_frac', 'dominant_score', 'label_score']
gc.collect()<create_dataframe>
df_train.isna().sum()
train_generator = Landmark2020_DataGenerator(pathsTrain, labelsTrain, batch_size, shape, use_cache=False, augment = False, shuffle = True)
 val_generator = Landmark2020_DataGenerator(pathsVal, labelsVal, batch_size, shape, use_cache=False, shuffle = False )<init_hyperparams>
def log_transform(feature, df_train, df_test):
min_feature = min(df_train[feature].min() , df_test[feature].min())
 df_train[feature] = np.log(df_train[feature] + 1 - min_feature)
 df_test[feature] = np.log(df_test[feature] + 1 - min_feature)
 return df_train, df_test
epochs = 1
use_multiprocessing = True 
<train_model>
df_train, df_test = log_transform('vertex_x', df_train, df_test )
model.fit_generator(
 train_generator,
steps_per_epoch=labelsTrain.shape[0]/batch_size,
validation_data=val_generator,
validation_steps=labelsVal.shape[0]/batch_size,
epochs=epochs,
use_multiprocessing=use_multiprocessing,
verbose=1 )<set_options>
df_train, df_test = log_transform('vertex_y', df_train, df_test )
gc.collect()<set_options>
df_train['bounding_agg'] =(df_train.bounding_confidence + df_train.bounding_importance)/ 2
df_test['bounding_agg'] =(df_test.bounding_confidence + df_test.bounding_importance)/ 2
gc.collect()<predict_on_test>
df_train['dominant_color'] =(df_train.dominant_blue + df_train.dominant_green + df_train.dominant_red)/ 3
df_test['dominant_color'] =(df_test.dominant_blue + df_test.dominant_green + df_test.dominant_red)/ 3
test_img = skimage.io.imread('.. /input/landmark-retrieval-2020/train/0/0/0/000014b1f770f640.jpg')
 print(test_img.shape)
 print(type(test_img))
y = model.predict(test_img.reshape(1,test_img.shape[0],test_img.shape[1],3))
y<train_model>
df_train, df_test = log_transform('dominant_color', df_train, df_test )
class MyModel(tf.keras.Model):
def __init__(self):
super(MyModel, self ).__init__()
self.model = embedding_model
@tf.function(input_signature=[
tf.TensorSpec(shape=[None, None, 3], dtype=tf.uint8, name='input_image')
 ])
 def call(self, input_image):
output_tensors = {}
im = tf.image.resize(input_image,(IMAGE_SIZE, IMAGE_SIZE))
extracted_features = self.model(tf.convert_to_tensor([im], dtype=tf.uint8)) [0]
output_tensors['global_descriptor'] = tf.identity(extracted_features, name='global_descriptor')
 return output_tensors<save_model>
df_train['dominant_frac_agg'] =(df_train.dominant_pixel_frac + df_train.dominant_score + df_train.label_score)/ 3
df_test['dominant_frac_agg'] =(df_test.dominant_pixel_frac + df_test.dominant_score + df_test.label_score)/ 3
m = MyModel()
served_function = m.call
tf.saved_model.save(
 m,
export_dir="./model",
signatures={'serving_default': served_function}
)<load_pretrained>
df_train.drop(['Name', 'Description', 'RescuerID', 'bounding_confidence', 'bounding_importance', 'dominant_blue', 'dominant_green', 'dominant_red', 'dominant_pixel_frac', 'dominant_score', 'label_score', 'label_description'], axis=1, inplace=True)
 df_test.drop(['Name', 'Description', 'RescuerID', 'bounding_confidence', 'bounding_importance', 'dominant_blue', 'dominant_green', 'dominant_red', 'dominant_pixel_frac', 'dominant_score', 'label_score', 'label_description'], axis=1, inplace=True )
with ZipFile('submission.zip','w')as output_zip_file:
for filename in os.listdir('./model'):
if os.path.isfile('./model/'+filename):
output_zip_file.write('./model/'+filename, arcname=filename)
 for filename in os.listdir('./model/variables'):
if os.path.isfile('./model/variables/'+filename):
output_zip_file.write('./model/variables/'+filename, arcname='variables/'+filename)
 for filename in os.listdir('./model/assets'):
if os.path.isfile('./model/assets/'+filename):
output_zip_file.write('./model/assets/'+filename, arcname='assets/'+filename )<load_from_csv>
magnitude_std = df_train.desc_magnitude.std()
magnitude_mean = df_train.desc_magnitude.mean()
score_std = df_train.desc_score.std()
score_mean = df_train.desc_score.mean()
df_train['desc_magnitude'].fillna(np.random.normal(magnitude_mean, magnitude_std), inplace=True)
 df_train['desc_score'].fillna(np.random.normal(score_mean, score_std), inplace=True)
 df_test['desc_magnitude'].fillna(np.random.normal(magnitude_mean, magnitude_std), inplace=True)
 df_test['desc_score'].fillna(np.random.normal(score_mean, score_std), inplace=True )
image_ids = pd.read_csv(
 '.. /input/landmark-retrieval-2020/train.csv',
nrows=100
)<train_model>
category_columns = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2', 'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed', 'Sterilized', 'Health', 'State']
numerical_columns = ['Age', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'name_len', 'desc_magnitude', 'desc_score', 'bounding_agg', 'dominant_color', 'dominant_frac_agg']
def get_image(img_id):
chars = [char for char in img_id]
dir_1, dir_2, dir_3 = chars[0], chars[1], chars[2]
image = Image.open('.. /input/landmark-retrieval-2020/train/' + dir_1 + '/' + dir_2 + '/' + dir_3 + '/' + img_id + '.jpg')
 image = image.resize(( IMAGE_SIZE, IMAGE_SIZE))
image = np.asarray(image)/ 255.0
return image<define_variables>
df_train[category_columns] = df_train[category_columns].astype('category')
 df_test[category_columns] = df_test[category_columns].astype('category' )
images = [get_image(img_id)for img_id in image_ids.id]
images = np.array(images )<predict_on_test>
min_age = min(df_train.Age.min() , df_test.Age.min())
 df_train.Age = np.log(df_train.Age + 1 - min_age)
 df_test.Age = np.log(df_test.Age + 1 - min_age )
embeddings = embedding_model.predict(images )<train_model>
min_magn = min(df_train.desc_magnitude.min() , df_test.desc_magnitude.min())
 df_train.desc_magnitude = np.log(df_train.desc_magnitude + 1 - min_magn)
 df_test.desc_magnitude = np.log(df_test.desc_magnitude + 1 - min_magn )
distances = distance.cdist(embeddings, embeddings, 'euclidean')
 print(distances.shape )<define_variables>
df_train.name_len = np.log(df_train.name_len + 1)
 df_test.name_len = np.log(df_test.name_len + 1 )
<set_options>
from sklearn.model_selection import train_test_split
init_notebook_mode(connected=True)
 print(os.listdir(".. /input"))<load_from_csv>
from sklearn.model_selection import train_test_split
df_id = pd.read_csv(".. /input/train_identity.csv")
 df_trans = pd.read_csv(".. /input/train_transaction.csv" )<statistical_test>
X_train, X_val, y_train, y_val = train_test_split(df_train, target, train_size=0.8, random_state=1234 )
def resumetable(df):
print(f"Dataset Shape: {df.shape}")
 summary = pd.DataFrame(df.dtypes,columns=['dtypes'])
 summary = summary.reset_index()
summary['Name'] = summary['index']
summary = summary[['Name','dtypes']]
summary['Missing'] = df.isnull().sum().values
summary['Uniques'] = df.nunique().values
summary['First Value'] = df.loc[0].values
summary['Second Value'] = df.loc[1].values
summary['Third Value'] = df.loc[2].values
for name in summary['Name'].value_counts().index:
summary.loc[summary['Name'] == name, 'Entropy'] = round(stats.entropy(df[name].value_counts(normalize=True), base=2),2)
 return summary
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
def CalcOutliers(df_num):
data_mean, data_std = np.mean(df_num), np.std(df_num)
 cut = data_std * 3
lower, upper = data_mean - cut, data_mean + cut
outliers_lower = [x for x in df_num if x < lower]
outliers_higher = [x for x in df_num if x > upper]
outliers_total = [x for x in df_num if x < lower or x > upper]
outliers_removed = [x for x in df_num if x > lower and x < upper]
print('Identified lowest outliers: %d' % len(outliers_lower))
print('Identified upper outliers: %d' % len(outliers_higher))
print('Total outlier observations: %d' % len(outliers_total))
print('Non-outlier observations: %d' % len(outliers_removed))
print("Total percentual of Outliers: ", round(( len(outliers_total)/ len(outliers_removed)) *100, 4))
return<drop_column>
import lightgbm as lgbm
df_trans = reduce_mem_usage(df_trans)
 df_id = reduce_mem_usage(df_id )<data_type_conversions>
params_lgbm = {'num_leaves': 38,
'min_data_in_leaf': 146,
'objective':'multiclass',
'num_class': 5,
'max_depth': 4,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9980062052116254,
"bagging_freq": 1,
"bagging_fraction": 0.844212672233457,
"bagging_seed": 11,
"metric": 'multi_logloss',
"lambda_l1": 0.12757257166471625,
"random_state": 133,
"verbosity": -1
}
df_trans['TransactionAmt'] = df_trans['TransactionAmt'].astype(float)
 print("Transaction Amounts Quantiles:")
 print(df_trans['TransactionAmt'].quantile([.01,.025,.1,.25,.5,.75,.9,.975,.99]))<define_search_space>
lgbm_train = lgbm.Dataset(X_train, y_train, categorical_feature=category_columns)
 lgbm_valid = lgbm.Dataset(X_val, y_val, categorical_feature=category_columns )
print("Card Features Quantiles: ")
 print(df_trans[['card1', 'card2', 'card3', 'card5']].quantile([0.01,.025,.1,.25,.5,.75,.975,.99]))<feature_engineering>
model_lgbm = lgbm.train(params_lgbm, lgbm_train, 10000, valid_sets=[lgbm_valid], verbose_eval= 500, categorical_feature=category_columns, early_stopping_rounds = 200 )
df_trans.loc[df_trans.card3.isin(df_trans.card3.value_counts() [df_trans.card3.value_counts() < 200].index), 'card3'] = "Others"
df_trans.loc[df_trans.card5.isin(df_trans.card5.value_counts() [df_trans.card5.value_counts() < 300].index), 'card5'] = "Others"<define_search_space>
( np.argmax(model_lgbm.predict(X_val), axis=1)== y_val ).sum() / y_val.shape[0]
print("Card Features Quantiles: ")
 print(df_trans[['addr1', 'addr2']].quantile([0.01,.025,.1,.25,.5,.75,.90,.975,.99]))<feature_engineering>
from sklearn.ensemble import RandomForestClassifier as Rf
from sklearn.ensemble import GradientBoostingClassifier as Gb
df_trans.loc[df_trans.addr1.isin(df_trans.addr1.value_counts() [df_trans.addr1.value_counts() <= 5000 ].index), 'addr1'] = "Others"
df_trans.loc[df_trans.addr2.isin(df_trans.addr2.value_counts() [df_trans.addr2.value_counts() <= 50 ].index), 'addr2'] = "Others"<feature_engineering>
model_rf = Rf()
df_trans.loc[df_trans['P_emaildomain'].isin(['gmail.com', 'gmail']),'P_emaildomain'] = 'Google'
df_trans.loc[df_trans['P_emaildomain'].isin(['yahoo.com', 'yahoo.com.mx', 'yahoo.co.uk',
'yahoo.co.jp', 'yahoo.de', 'yahoo.fr',
'yahoo.es']), 'P_emaildomain'] = 'Yahoo Mail'
df_trans.loc[df_trans['P_emaildomain'].isin(['hotmail.com','outlook.com','msn.com', 'live.com.mx',
'hotmail.es','hotmail.co.uk', 'hotmail.de',
'outlook.es', 'live.com', 'live.fr',
'hotmail.fr']), 'P_emaildomain'] = 'Microsoft'
df_trans.loc[df_trans.P_emaildomain.isin(df_trans.P_emaildomain\ 
.value_counts() [df_trans.P_emaildomain.value_counts() <= 500 ]\ 
.index), 'P_emaildomain'] = "Others"
df_trans.P_emaildomain.fillna("NoInf", inplace=True )<concatenate>
model_rf.fit(X_train,y_train )
ploting_cnt_amt(df_trans, 'P_emaildomain' )<feature_engineering>
model_rf.score(X_train,y_train )
df_trans.loc[df_trans['R_emaildomain'].isin(['gmail.com', 'gmail']),'R_emaildomain'] = 'Google'
df_trans.loc[df_trans['R_emaildomain'].isin(['yahoo.com', 'yahoo.com.mx', 'yahoo.co.uk',
'yahoo.co.jp', 'yahoo.de', 'yahoo.fr',
'yahoo.es']), 'R_emaildomain'] = 'Yahoo Mail'
df_trans.loc[df_trans['R_emaildomain'].isin(['hotmail.com','outlook.com','msn.com', 'live.com.mx',
'hotmail.es','hotmail.co.uk', 'hotmail.de',
'outlook.es', 'live.com', 'live.fr',
'hotmail.fr']), 'R_emaildomain'] = 'Microsoft'
df_trans.loc[df_trans.R_emaildomain.isin(df_trans.R_emaildomain\ 
.value_counts() [df_trans.R_emaildomain.value_counts() <= 300 ]\ 
.index), 'R_emaildomain'] = "Others"
df_trans.R_emaildomain.fillna("NoInf", inplace=True )<concatenate>
model_rf.score(X_val,y_val )
ploting_cnt_amt(df_trans, 'R_emaildomain' )<feature_engineering>
model_gb = Gb()
df_trans.loc[df_trans.C1.isin(df_trans.C1\ 
.value_counts() [df_trans.C1.value_counts() <= 400 ]\ 
.index), 'C1'] = "Others"<count_values>
model_gb.fit(X_train,y_train )
df_trans.loc[df_trans.C2.isin(df_trans.C2\ 
.value_counts() [df_trans.C2.value_counts() <= 350 ]\ 
.index), 'C2'] = "Others"<concatenate>
model_gb.score(X_train,y_train )
ploting_cnt_amt(df_trans, 'C2' )<feature_engineering>
model_gb.score(X_val,y_val )
START_DATE = '2017-12-01'
startdate = datetime.datetime.strptime(START_DATE, "%Y-%m-%d")
 df_trans["Date"] = df_trans['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds=x)))
 df_trans['_Weekdays'] = df_trans['Date'].dt.dayofweek
df_trans['_Hours'] = df_trans['Date'].dt.hour
df_trans['_Days'] = df_trans['Date'].dt.day<groupby>
val_lgbm = model_lgbm.predict(X_val)
 val_gb = model_gb.predict_log_proba(X_val )
color_op = ['
'
'
dates_temp = df_trans.groupby(df_trans.Date.dt.date)['TransactionAmt'].count().reset_index()
trace = go.Scatter(x=dates_temp['Date'], y=dates_temp.TransactionAmt,
opacity = 0.8, line = dict(color = color_op[7]), name= 'Total Transactions')
 dates_temp_sum = df_trans.groupby(df_trans.Date.dt.date)['TransactionAmt'].sum().reset_index()
trace1 = go.Scatter(x=dates_temp_sum.Date, line = dict(color = color_op[1]), name="Total Amount",
y=dates_temp_sum['TransactionAmt'], opacity = 0.8, yaxis='y2')
 layout = dict(
 title= "Total Transactions and Fraud Informations by Date",
xaxis=dict(
 rangeselector=dict(
 buttons=list([
dict(count=1, label='1m', step='month', stepmode='backward'),
dict(count=3, label='3m', step='month', stepmode='backward'),
dict(count=6, label='6m', step='month', stepmode='backward'),
dict(step='all')
 ])
),
rangeslider=dict(visible = True),
type='date'),
yaxis=dict(title='Total Transactions'),
yaxis2=dict(overlaying='y',
anchor='x', side='right',
zeroline=False, showgrid=False,
title='Total Transaction Amount')
)
 fig = dict(data= [trace, trace1,], layout=layout)
 iplot(fig )<groupby>
val_mixed =(val_gb + val_lgbm)/ 2
color_op = ['
'
'
tmp_amt = df_trans.groupby([df_trans.Date.dt.date, 'isFraud'])['TransactionAmt'].sum().reset_index()
tmp_trans = df_trans.groupby([df_trans.Date.dt.date, 'isFraud'])['TransactionAmt'].count().reset_index()
tmp_trans_fraud = tmp_trans[tmp_trans['isFraud'] == 1]
tmp_amt_fraud = tmp_amt[tmp_amt['isFraud'] == 1]
dates_temp = df_trans.groupby(df_trans.Date.dt.date)['TransactionAmt'].count().reset_index()
trace = go.Scatter(x=tmp_trans_fraud['Date'], y=tmp_trans_fraud.TransactionAmt,
opacity = 0.8, line = dict(color = color_op[1]), name= 'Fraud Transactions')
 trace1 = go.Scatter(x=tmp_amt_fraud.Date, line = dict(color = color_op[7]), name="Fraud Amount",
y=tmp_amt_fraud['TransactionAmt'], opacity = 0.8, yaxis='y2')
 layout = dict(
 title= "FRAUD TRANSACTIONS - Total Transactions and Fraud Informations by Date",
xaxis=dict(
 rangeselector=dict(
 buttons=list([
dict(count=1, label='1m', step='month', stepmode='backward'),
dict(count=3, label='3m', step='month', stepmode='backward'),
dict(count=6, label='6m', step='month', stepmode='backward'),
dict(step='all')
 ])
),
rangeslider=dict(visible = True),
type='date'),
yaxis=dict(title='Total Transactions'),
yaxis2=dict(overlaying='y',
anchor='x', side='right',
zeroline=False, showgrid=False,
title='Total Transaction Amount')
)
 fig = dict(data= [trace, trace1], layout=layout)
 iplot(fig )<merge>
( np.argmax(val_mixed, axis=1)== y_val ).sum() / y_val.shape[0]
df_train = df_trans.merge(df_id, how='left', left_index=True, right_index=True )<feature_engineering>
test_lgbm = np.argmax(model_lgbm.predict(df_test), axis=1 )
df_train.loc[df_train['id_30'].str.contains('Windows', na=False), 'id_30'] = 'Windows'
df_train.loc[df_train['id_30'].str.contains('iOS', na=False), 'id_30'] = 'iOS'
df_train.loc[df_train['id_30'].str.contains('Mac OS', na=False), 'id_30'] = 'Mac'
df_train.loc[df_train['id_30'].str.contains('Android', na=False), 'id_30'] = 'Android'
df_train['id_30'].fillna("NAN", inplace=True )<feature_engineering>
test_id = pd.DataFrame(test_id )
df_train.loc[df_train['id_31'].str.contains('chrome', na=False), 'id_31'] = 'Chrome'
df_train.loc[df_train['id_31'].str.contains('firefox', na=False), 'id_31'] = 'Firefox'
df_train.loc[df_train['id_31'].str.contains('safari', na=False), 'id_31'] = 'Safari'
df_train.loc[df_train['id_31'].str.contains('edge', na=False), 'id_31'] = 'Edge'
df_train.loc[df_train['id_31'].str.contains('ie', na=False), 'id_31'] = 'IE'
df_train.loc[df_train['id_31'].str.contains('samsung', na=False), 'id_31'] = 'Samsung'
df_train.loc[df_train['id_31'].str.contains('opera', na=False), 'id_31'] = 'Opera'
df_train['id_31'].fillna("NAN", inplace=True)
 df_train.loc[df_train.id_31.isin(df_train.id_31.value_counts() [df_train.id_31.value_counts() < 200].index), 'id_31'] = "Others"<load_from_csv>
submission = test_id.join(pd.DataFrame(test_lgbm, columns=['AdoptionSpeed']))
df_trans = pd.read_csv('.. /input/train_transaction.csv')
 df_test_trans = pd.read_csv('.. /input/test_transaction.csv')
 df_id = pd.read_csv('.. /input/train_identity.csv')
 df_test_id = pd.read_csv('.. /input/test_identity.csv')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 df_train = df_trans.merge(df_id, how='left', left_index=True, right_index=True, on='TransactionID')
 df_test = df_test_trans.merge(df_test_id, how='left', left_index=True, right_index=True, on='TransactionID')
 print(df_train.shape)
 print(df_test.shape)
 del df_trans, df_id, df_test_trans, df_test_id 
<drop_column>
submission.to_csv('submission.csv', index=False )
<define_variables><EOS> 
submission.to_csv('submission.csv', index=False )