Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<load_from_csv>
leak_df = reduce_mem_usage(pd.read_csv('/kaggle/input/leakaggregator/leaked_test_target.csv'))
not_nan_values_in_leak_df = ~leak_df['meter_reading'].isna()<count_values>
print('Finished importing.... ')
 modeling_start_time = time.time()
print('******Modeling start time******:' , 
(time.strftime("%Hhrs:%Mm:%Ss", time.gmtime(( modeling_start_time)))))
 print('Loading data.... ')
 print('Setting opition pd.set_option display.max_colwidth to avoid truncation of columns.... ')
 pd.set_option('display.max_colwidth', -1)
 origtrain = pd.read_table('.. /input/train.tsv',
converters = {'brand_name': np.str ,
'category_name': np.str,
'item_condition_id': np.uint8,
'item_description': np.str,
'name': np.str,
'price': np.float64,
'shipping': np.uint8,
'train_id' : np.uint64
})
 origtest = pd.read_table('.. /input/test.tsv',
converters = {'brand_name': np.str ,
'category_name': np.str,
'item_condition_id': np.uint8,
'item_description': np.str,
'name': np.str,
'price': np.float64,
'shipping': np.uint8,
'train_id' : np.uint64
})
 print(" The number of rows in train", origtrain.shape)
 print(" The number of rows in test", origtest.shape)
 print(" Number of rows with price zero : " ,origtrain.price[origtrain.price == 0.0].count())
 print(" Number of rows : ", origtrain.shape[0])
 origtrain = origtrain[origtrain.price > 0.0]
nrow_train = origtrain.shape[0]
print(" Number of rows aferdrop : ", nrow_train)
 test_id = origtest.test_id.values
combdata = pd.concat([origtrain , origtest],0)
 combdata = combdata.reindex()
stop_words = set(stopwords.words('english'))
print('Finished loading data.... ')
 print
del origtrain
del origtest
gc.collect()
not_nan_values_in_leak_df.value_counts(normalize=True )<define_variables>
start_time = time.time()
print('Starting pre-processing of text columns at start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss", time.gmtime(( start_time)))))
 print(' pre-processing brand name...')
 combdata.brand_name = combdata.brand_name.apply(removeSpecial)
 combdata.loc[combdata['brand_name'].values == '', 'brand_name'] = '0000aaaa'
combdata['newBrand'] = combdata['brand_name'].str.strip()
print(' pre-processing price...')
 combdata.price = combdata.price.fillna(0)
 combdata.price = combdata.price.astype('float16')
 print(' pre-processing shipping...')
 combdata.shipping = combdata.shipping.astype('uint8')
 print(' pre-processing item condition...')
 combdata.item_condition_id = combdata.item_condition_id.astype('uint8')
 print(' pre-processing category name...')
 combdata.loc[combdata['category_name'].values == '', 'category_name'] = '0000aaaa'
combdata['newCategory'] = combdata.category_name.apply(removeSpecial)
 print(' pre-processing test_id...')
 combdata.test_id = combdata.test_id.fillna(9999999)
 combdata.test_id = combdata.test_id.astype('uint32')
 print(' pre-processing train id...')
 combdata.train_id = combdata.train_id.fillna(9999999)
 combdata.train_id = combdata.train_id.astype('uint32')
 print(' filling the null with 0000aaaa...')
 combdata.loc[combdata['name'].values == '', 'name'] = '0000aaaa'
combdata.loc[combdata['item_description'].values == '', 'item_description'] = '0000aaaa'
print(' creating new columns newName and description...')
 combdata['newName'] = combdata.name.values
combdata['description'] = combdata.item_description.values
print(' removing special character from newName...')
 combdata['newName'] = combdata.newName.apply(removeSpecial)
 print(' removing special character from description...')
 combdata['description'] = combdata.description.apply(removeSpecial)
 end_time = end_time = time.time()
print('Completed pre-processing of text columns and time taken:', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))
gc.collect()
datasets_to_take = [
'/kaggle/input/ashrae-submissions/*.csv',
'/kaggle/input/ashrae-submissions-1/*.csv',
'/kaggle/input/ashrae-submissions-2/*.csv',
'/kaggle/input/ashrae-submissions-3/*.csv',
'/kaggle/input/ashrae-submissions-4/*.csv',
'/kaggle/input/ashrae-submissions-5/*.csv',
'/kaggle/input/ashrae-submissions-6/*.csv'
]
files_to_take = []
for dtt in datasets_to_take:
files_to_take += glob(dtt )<feature_engineering>
start_time = time.time()
print('Building sentneces start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss", time.gmtime(( start_time)))))
 sentences = []
mask =(combdata['description'].values != 'no description yet')&(combdata['description'].values != '0000 aaaa')
 cmask =(combdata['newCategory'].values != '0000aaaa')
 bmask =(combdata['newBrand'].values != '0000aaaa')
 vtext = [ [x] for x in combdata.description[mask].values]
vtext.extend([ [x] for x in combdata.newCategory[cmask].values])
 vtext.extend([ [x] for x in combdata.newBrand[bmask].values ])
 vtext.extend([ [x] for x in combdata.newName.values ])
 for sent in vtext:
for words in sent :
w = [ word for word in words.split(' ')if(( len(word)> 1)&(word not in stop_words)) ]
sentences.append(w)
 print(' The raw_text type : %s and no of words : %d
sample : %s'
%(type(sentences), len(sentences), sentences[0:1]))
end_time = time.time()
print('Completed building sentneces and time taken::', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))
del vtext
gc.collect()
all_sumbissions = pd.DataFrame()
all_sumbissions['row_id'] = leak_df['row_id']
if ADD_LIST:
print('ADD MODE')
 for df_name in tqdm(files_to_take):
if os.path.basename(df_name ).split('.')[0] in ADD_LIST:
print(df_name)
 all_sumbissions[os.path.basename(df_name ).split('.')[0]] = reduce_mem_usage(pd.read_csv(df_name)).sort_values('row_id')['meter_reading']
gc.collect()
else:
print('EXCLUDE MODE')
 for df_name in tqdm(files_to_take):
if os.path.basename(df_name ).split('.')[0] in EXCLUDE_LIST:
continue
print(df_name)
 all_sumbissions[os.path.basename(df_name ).split('.')[0]] = reduce_mem_usage(pd.read_csv(df_name)).sort_values('row_id')['meter_reading']
gc.collect()<set_options>
noOfOccurance = 10
wordVecSize = 200
windowSize = 5
noOfWorkers = 8
vocab_size = len(sentences)
 start_time = time.time()
print('Building the Word2Vec model at start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss", time.gmtime(( start_time)))))
 word2vecModel = word2vec.Word2Vec(sentences, size=wordVecSize, window=windowSize,
min_count=noOfOccurance, workers=noOfWorkers)
 word2vecModel.init_sims(replace=True)
 gc.collect()
end_time = time.time()
print('Completed building word2vec word2vecModel and time taken::', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))
gc.collect()<feature_engineering>
start_time = time.time()
print('Building tfidf score matrix for the vocab and and start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss", time.gmtime(( start_time)))))
 noOfocurance = 10
vectorizer = TfidfVectorizer(analyzer=lambda x: x, min_df=noOfocurance)
 matrix = vectorizer.fit_transform([x for x in sentences])
 tfidf = dict(zip(vectorizer.get_feature_names() , vectorizer.idf_))
print(' vocab size(%d )...' %(len(tfidf)))
 end_time = time.time()
print('Completed tfidf score matrix for the vocab and time taken::', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))
for col in all_sumbissions.columns[all_sumbissions.min() < 0]:
all_sumbissions.loc[all_sumbissions[col] < 0, col] = 0<set_options>
def getBrand(text):
global globvar
global nameString
global nameDesc
quit = False
if(nameDesc):
nameText = nameString[globvar]
else:
nameText = descString[globvar]
globvar += 1
words = nameText.split()
rtext ='0000aaaa'
for word in words:
brandFound = brandNames.get(word, 'NO')
 if brandFound != 'NO':
rtext = brandFound
quit = True
break
if not quit:
for brand in brandMultiNames.keys() :
result = ''
result = re.search('\\b'+brand+'\\b', nameText)
 if(result):
rtext = brand
quit = True
break
return(rtext )
gc.collect()<compute_train_metric>
start_time = time.time()
print('Starting replacing null brands and start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( start_time))))))
globvar = 0
nameString = []
descString = []
nameDesc = True
mask =(combdata['newBrand'].values == '0000aaaa')
 noOfNull = combdata.newBrand[mask].values
nameString = combdata.newName[mask].values
print(' The count of brands with 0000aaaa are %d Before searching through newName' %(len(noOfNull)))
 combdata['Brand'] = combdata['newBrand'].apply(lambda x: x if x != '0000aaaa' else getBrand(x))
mask = combdata['Brand'].values == '0000aaaa'
noOfNull = combdata.newBrand[mask]
print(' The count of brands with 0000aaaa are %d After searching through newName' %(len(noOfNull)))
 globvar = 0
descString = []
nameDesc = False
mask =(( combdata['Brand'].values == '0000aaaa'))
btext = combdata.Brand[mask].values
descString = combdata.description[mask].values
print(' The count of brands with 0000aaaa are %d Before searching through description' %(len(btext)))
 combdata['Brand'] = combdata['Brand'].apply(lambda x: x if x != '0000aaaa' else getBrand(x))
mask = combdata['Brand'].values == '0000aaaa'
x = combdata.newBrand[mask]
print(' The count of brands with 0000aaaa are %d After searching through description' %(len(x)))
 del nameString
del descString
del btext
del mask
gc.collect()
end_time = time.time()
print('Completed replacing null brands and time taken to process:', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))
for col in all_sumbissions.columns[1:]:
print(col)
 leak_score = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'],
all_sumbissions.loc[not_nan_values_in_leak_df, col])
 print('score1=', leak_score)
 gc.collect()<compute_train_metric>
start_time = time.time()
print('Creating new feature len_description and len_name and start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( start_time))))))
print(' Creating new feature len_description for column description...')
 try:
combdata['len_description'] = combdata['description'].map(len)
 except TypeError as e:
print(e)
 x = np.max(combdata['len_description'])
 print(' Creating new feature len_name for column newName...')
 try:
combdata['len_name'] = combdata['newName'].map(len)
 except TypeError as e:
print(e)
 y = np.max(combdata['len_name'])
 print(' max length of column newName is(%d)and column description is(%d)e...' %(y,x))
end_time = end_time = time.time()
print('Completed creating new feature len_description and len_name and time taken::', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))
leak_score = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'], all_sumbissions.loc[not_nan_values_in_leak_df].iloc[:,1:].mean(axis=1))
print('mean score=', leak_score )<compute_train_metric>
start_time = time.time()
print('Encoding of item_condition_id and shipping and start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( start_time))))))
combdata['hot_shp_item_cnd'] = csr_matrix(pd.get_dummies(combdata[["item_condition_id", "shipping" ]], sparse=True ).values)
 print(' Shape of X_shp_item_cnd after encoding %s...'%(str(combdata['hot_shp_item_cnd'].shape)))
 end_time = end_time = time.time()
print('Completed encoding of item_condition_id and shipping and time taken::', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))
leak_score = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'], all_sumbissions.loc[not_nan_values_in_leak_df].iloc[:,1:].median(axis=1))
print('mean score=', leak_score )<train_model>
start_time = time.time()
print('Starting encoding Brand and newCategory and start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( start_time))))))
le = LabelEncoder()
le.fit(np.hstack([combdata.newCategory]))
combdata['lbl_category'] = le.transform(combdata.newCategory)
 le.fit(np.hstack([combdata.Brand]))
combdata['lbl_brand_name'] = le.transform(combdata.Brand)
 del le
gc.collect()
end_time = time.time()
print('Completed encoding Brand and newCategory and time taken to process:', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))
class GenerativeAlgorithm(object):
def __init__(self,
x_shape,
amout_ind_to_mutate,
population_size,
target_function,
num_generations,
populations_to_take,
available_indices,
mode='min'):
self.amout_ind_to_mutate = amout_ind_to_mutate
self.x_shape = x_shape
self.population_size = population_size
self.target_function = target_function
self.num_generations = num_generations
self.populations_to_take = populations_to_take
self.available_indices = available_indices
self.mode = mode
self.gen_initial_generation()
if self.mode == 'min':
self.best_score = 1000000
if self.mode == 'max':
self.best_score = -1000000
def gen_initial_generation(self):
self.initial_generation = []
for i, el in enumerate(permutations(self.available_indices, self.x_shape)) :
if i == self.population_size:
break
self.initial_generation.append(list(el))
self.initial_generation = np.array(self.initial_generation)
 def select_mating_pool(self, population, fitness):
parents = np.empty(( self.populations_to_take, population.shape[1]))
for parent_num in range(self.populations_to_take):
if self.mode == 'min':
fitness_idx = np.argmin(fitness)
 elif self.mode == 'max':
fitness_idx = np.argmax(fitness)
 else:
raise ValueError('Not implemented!')
 parents[parent_num, :] = population[fitness_idx, :]
if self.mode == 'min':
fitness[fitness_idx] = 1000000
elif self.mode == 'max':
fitness[fitness_idx] = - 1000000
return parents
def crossover(self, parents, offspring_size):
offspring = np.empty(offspring_size)
 crossover_point = np.uint8(offspring_size[1]/2)
 for k in range(offspring_size[0]):
parent1_idx = np.random.randint(0, parents.shape[0])
 parent2_idx = np.random.randint(0, parents.shape[0])
 offspring[k, :crossover_point] = parents[parent1_idx, :crossover_point]
offspring[k, crossover_point:] = parents[parent2_idx, crossover_point:]
return offspring
def mutation(self, offspring_crossover):
for idx in range(offspring_crossover.shape[0]):
for i in range(self.amout_ind_to_mutate):
cur_available_idices = list(set(self.available_indices)- set(offspring_crossover[idx,:]))
value = np.random.choice(cur_available_idices)
 idx_1 = np.random.randint(0,offspring_crossover.shape[1])
 offspring_crossover[idx, idx_1] = value
return offspring_crossover
def apply_function_on_array(self, ar):
return np.array([self.target_function(ar[i,:])for i in range(ar.shape[0])])
 def fit(self):
cur_population = self.initial_generation
history = {'best_points':[], 'best_fits':[]}
generation = 0
while generation < self.num_generations:
fitness = self.apply_function_on_array(cur_population)
 if self.mode == 'min':
best_idx = np.argmin(fitness)
 elif self.mode == 'max':
best_idx = np.argmax(fitness)
 history['best_points'].append(cur_population[best_idx,:])
 history['best_fits'].append(fitness[best_idx])
 self.best_score = fitness[best_idx]
print('Itteration {} best score: {}'.format(generation, self.best_score))
parents = self.select_mating_pool(cur_population, fitness)
 offspring_crossover = self.crossover(parents,
offspring_size=(self.population_size-parents.shape[0], cur_population.shape[1]))
offspring_mutation = self.mutation(offspring_crossover)
 cur_population[:parents.shape[0], :] = parents
cur_population[parents.shape[0]:, :] = offspring_mutation
generation+=1
fitness = self.apply_function_on_array(cur_population)
 if self.mode == 'min':
best_idx = np.argmin(fitness)
 elif self.mode == 'max':
best_idx == np.argmax(fitness)
 history['best_points'].append(cur_population[best_idx,:])
 history['best_fits'].append(fitness[best_idx])
 return history<train_on_grid>
def buildWordVector(tokens, size, vec):
count = 0.
 for words in tokens.split(' '):
try:
vec = vec + word2vecModel.wv.get_vector(words ).reshape(( 1, size)) * tfidf[words]
count += 1.
 except KeyError:
continue
if count != 0:
vec /= count
return vec
if USE_GENETIC_ALG:
def tgt_f(col_idx):
sc = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'],
all_sumbissions.loc[not_nan_values_in_leak_df].iloc[:,col_idx].median(axis=1))
return sc
a_ind = np.array(range(1, all_sumbissions.shape[1]-1))
alg = GenerativeAlgorithm(
 amout_ind_to_mutate=6,
population_size=20,
x_shape=11,
target_function=tgt_f,
num_generations=2,
populations_to_take=6,
available_indices=a_ind 
)
 h = alg.fit()<find_best_params>
start_time = time.time()
print('Spliting train and test data and start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss", time.gmtime(( start_time)))))
 print('The original train no of rows(%d)' %(nrow_train))
org_train = combdata[:nrow_train]
X_test = combdata[nrow_train:]
print(' org_train shape ', org_train.shape)
 print(' X_test shape ', X_test.shape)
 print(' Spliting training data into train and validation set...')
 X_splt_train, X_splt_val = train_test_split(org_train, random_state=42, test_size=0.10)
 y_train = X_splt_train['price'].values
y_train = y_train.reshape(len(y_train),1)
 y_val = X_splt_val['price'].values
y_val = y_val.reshape(len(y_val),1)
 X_TRAIN_SHAPE = X_splt_train.shape[1]
Y_TRAIN_SHAPE = y_train.shape[1]
X_VAL_SHAPE = X_splt_val.shape[1]
Y_VAL_SHAPE = y_val.shape[1]
print(' X_splt_train shape ', X_splt_train.shape)
 print(' X_splt_val shape ', X_splt_val.shape)
 print(' y_train shape ', y_train.shape)
 print(' y_val shape ', y_val.shape)
 del combdata
gc.collect()
end_time = time.time()
print('Completed spliting train and test data and time taken::', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))
if USE_GENETIC_ALG:
best_cols = h['best_points'][-1]
m = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'], all_sumbissions.loc[not_nan_values_in_leak_df].iloc[:,best_cols].median(axis=1))
print('best_comb: {}
best_metric: {}'.format(all_sumbissions.columns[best_cols], m))<compute_train_metric>
def get_tflearn_data(dataset, scaler):
print(' Creating data dictionary...')
 wordVecSize = 200
zeroVec = np.zeros(wordVecSize, dtype=np.float32 ).reshape(( 1, wordVecSize))
print(' Creating name feature...')
 name = np.concatenate([buildWordVector(z, wordVecSize, zeroVec)for z in dataset.newName], axis=0)
 print(' The shape of namefeature...', name.shape)
 print(' Creating description feature...')
 description = np.concatenate([buildWordVector(z, wordVecSize, zeroVec)for z in dataset.description], axis=0)
 print(' The shape of description feature...', description.shape)
 brand_name = np.array(dataset.lbl_brand_name.values)
 brand_name = brand_name.reshape(len(brand_name),1)
 category = np.array(dataset.lbl_category.values)
 category = category.reshape(len(category),1)
 hot_shp_item_cnd = np.array(dataset.item_condition_id.values)
 item_condition = np.array(dataset.item_condition_id.values)
 item_condition = item_condition.reshape(len(item_condition), 1)
 shipping = np.array(dataset.shipping.values)
 shipping = shipping.reshape(len(shipping), 1)
 print(' Scaling inputs features...')
 name = scaler.fit_transform(name)
 description = scaler.fit_transform(description)
 brand_name = scaler.fit_transform(brand_name)
 category = scaler.fit_transform(category)
 item_condition = scaler.fit_transform(item_condition)
 shipping = scaler.fit_transform(shipping)
 print(' Finished scaling inputs features...')
 X = {
'name': name,
'item_desc' : description,
'brand_name' : brand_name,
'category' : category,
'item_condition' : item_condition,
'shipping' : shipping
}
print(' Finished data dictionary...')
 return(X)
if USE_HYPEROPT:
score_comb = []
def objective(x):
cols_idx = [el[1] for el in x]
if len(set(cols_idx)) != len(cols_idx):
return 100
m = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'], all_sumbissions[not_nan_values_in_leak_df].iloc[:,cols_idx].median(axis=1))
gc.collect()
score_comb.append([cols_idx,m])
 return m
space = [(str(i),1 + hp.hp.randint(str(i), all_sumbissions.shape[1]-1)) for i in range(10)]
best = hp.fmin(objective, space, algo=hp.tpe.suggest, max_evals=1000 )<find_best_params>
def getModel(dataset):
print('Preparing the model...')
 tf.reset_default_graph()
wordVecSize = 200
name = input_data(shape= [None, wordVecSize], name="name")
 item_desc = input_data(shape= [None, wordVecSize] , name="item_desc")
 brand_name = input_data(shape= [None, 1], name="brand_name")
 category = input_data(shape= [None,1], name="category")
 item_condition = input_data(shape= [None, 1], name="item_condition")
 shipping = input_data(shape= [None, 1], name="shipping")
 flat_name = flatten(name)
 flat_item_desc = flatten(item_desc)
 net = merge([ flat_name,
flat_item_desc,
brand_name,
category,
item_condition,
shipping],
mode = 'concat', axis = 1)
 X_TRAIN_SHAPE = net.shape[1]
print(' Buildig the model...')
 print(' Shape of the final merge input dataset:',X_TRAIN_SHAPE)
 print(' Creating input layer...')
 net1 = fully_connected(net, X_TRAIN_SHAPE, activation='ReLU')
 net2 = fully_connected(net1, X_TRAIN_SHAPE, activation='ReLU')
 net3 = fully_connected(net2, 256, activation='ReLU')
 net4 = fully_connected(net3, 1, activation='linear')
 net5 = regression(net4, optimizer='Adam', loss='mean_square', metric='R2')
 model = DNN(net5)
 print(' Finished building model...')
 print('Finished preparing the model...')
 return(model )
if USE_HYPEROPT:
best_cols = pd.DataFrame(score_comb ).sort_values(1 ).iloc[0,0]
m = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'], all_sumbissions.loc[not_nan_values_in_leak_df].iloc[:,best_cols].median(axis=1))
print('best_comb: {}
best_metric: {}'.format(all_sumbissions.columns[best_cols], m))<feature_engineering>
start_time = time.time()
print('Preparing data definations and start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss", time.gmtime(( start_time)))))
 scaler = sklearn.preprocessing.RobustScaler()
print(' Preparing Train data set...')
 X_train_dict = get_tflearn_data(X_splt_train,scaler)
 print(' Preparing Validation data set...')
 X_val_dict = get_tflearn_data(X_splt_val,scaler)
 print(' Preparing Test data set...')
 X_test_dict = get_tflearn_data(X_test, scaler)
 print(' Scaling price feature through log...')
 y_train = np.log1p(y_train)
 y_val = np.log1p(y_val)
 del word2vecModel
del tfidf
del sentences
del X_splt_train
del X_splt_val
del X_test
gc.collect()
print('Finished data definations...')
 end_time = time.time()
print('Completed preparing data definations and time taken::', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))
if USE_STACKING:
for col in all_sumbissions.columns[1:]:
all_sumbissions[col] = np.log1p(all_sumbissions[col])
 leak_df['meter_reading'] = np.log1p(leak_df['meter_reading'] )<set_options>
def rmsle(y_pred, y_actual):
return np.sqrt(np.square(np.log(y_pred + 1)- np.log(y_actual + 1)).mean() )
gc.collect()<feature_engineering>
start_time = time.time()
print('Building and training the modeland start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss", time.gmtime(( start_time)))))
 modelR = getModel(X_train_dict)
 print('Training the model...')
 modelR.fit(X_train_dict, y_train, n_epoch=5,
batch_size=256, snapshot_step= 1000,
show_metric=True,run_id='tflMP')
 end_time = time.time()
print('Completed building and training the model and time taken::', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))
if USE_STACKING:
all_sumbissions['target'] = leak_df['meter_reading']
del leak_df
gc.collect()<split>
start_time = time.time()
print('Evaluating the model the modeland start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss", time.gmtime(( start_time)))))
 score = modelR.evaluate(X_val_dict, y_val, batch_size=256)
 print(' The validation data set accuracy is:', score)
 val_rmsle = 0
y_val_pred = modelR.predict(X_val_dict)
 val_rmsle = rmsle(y_val_pred, y_val)
 print(' The validation data set rmsle is(%f)'%(val_rmsle))
end_time = time.time()
print('Completed evaluation of the model and time taken::', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))
<drop_column><EOS> 
start_time = time.time()
print('Predicting and submitting results and start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss", time.gmtime(( start_time)))))
 y_pred = np.zeros(X_test_dict['name'].shape[0])
 y_pred = modelR.predict(X_test_dict)
 y_pred = np.expm1(y_pred)
 print(' preparing for submission...')
 pricePred = pd.DataFrame()
pricePred['test_id'] = test_id
pricePred['price'] = np.round(y_pred,2)
 pricePred.to_csv('Sample_sub.csv',index=False)
 print(' submitted...')
 end_time = time.time()
print('******Price recommended in ******:' , 
(time.strftime("%Hhrs:%Mm:%Ss", time.gmtime(( end_time - modeling_start_time)))) )