Kaggle competition: DMA Kaggle Challenge
model.save_model('lightgbm_model.txt' )<filter>
df_train = pd.read_csv('/kaggle/input/dma-fall19/yelp_train.csv')
 df_test = pd.read_csv('/kaggle/input/dma-fall19/yelp_test.csv' )
<lightgbm.basic.Booster at 0x2ae02380f98><define_variables>
df_train = df_train.dropna()
bst = lgb.Booster(model_file='lightgbm_model.txt' )<set_options>
x_train = df_train[['cool', 'funny', 'useful', 'user_average_stars',
'user_review_count', 'business_latitude', 'business_longitude',
'business_review_count', 'business_average_stars']]
x_test = df_test[['cool', 'funny', 'useful', 'user_average_stars',
'user_review_count', 'business_latitude', 'business_longitude',
'business_review_count', 'business_average_stars']]
y_train = df_train['is_good_rating']
np.random.seed(1)
 pd.set_option("display.max_rows", 300)
 gc.enable()
y_label = 'winPlacePerc'
x_categorical = ['matchType']
standard_modes = ['solo', 'duo', 'squad', 'solo-fpp', 'duo-fpp', 'squad-fpp']
x_numeric_columns = [
'DBNOs', 'assists', 'boosts', 'damageDealt', 'headshotKills', 'heals', 'killPlace', 'killPoints',
'killStreaks', 'kills', 'longestKill', 'matchDuration', 'rankPoints', 'revives', 'rideDistance',
'roadKills', 'swimDistance', 'teamKills', 'vehicleDestroys', 'walkDistance', 'weaponsAcquired',
'winPoints', 'numGroups', 'maxPlace'
]
idx_columns = ['Id', 'matchId', 'groupId']
epochs = 1
batch_size = 2 ** 13
train_nrows = None
validation_split=0.0
print("batch size: {}".format(batch_size))
print("Input dir.. /input: {}".format(os.listdir(".. /input")) )<categorify>
clf = DecisionTreeClassifier()
clf.fit(x_train, y_train)
 accuracy_score(clf.predict(x_train), y_train )
def add_features(df):
print('New columns added')
 match_group_by = df.groupby(['matchId', 'groupId'])[x_numeric_columns]
print('match_group_by done')
 match_group_mean = match_group_by.mean()
print('match_group_mean done')
 match_group_mean_rank = match_group_mean.groupby('matchId')[x_numeric_columns].rank(pct=True)
 df = pd.merge(df, match_group_mean.reset_index() , on=['matchId', 'groupId'], how='left', suffixes=["", "_group_mean"], copy=False)
 df = pd.merge(df, match_group_mean_rank, on=['matchId', 'groupId'], how='left', suffixes=["", "_group_mean_rank"], copy=False)
 del match_group_mean, match_group_mean_rank; gc.collect()
print('all match_group_mean done')
 match_group_median = match_group_by.median()
match_group_median_rank = match_group_median.groupby('matchId')[x_numeric_columns].rank(pct=True)
 df = pd.merge(df, match_group_median.reset_index() , on=['matchId', 'groupId'], how='left', suffixes=["", "_group_median"], copy=False)
 df = pd.merge(df, match_group_median_rank, on=['matchId', 'groupId'], how='left', suffixes=["", "_group_median_rank"], copy=False)
 del match_group_median, match_group_median_rank; gc.collect()
print('match_group_median done')
 df = optimize_memory(df)
 match_group_size = match_group_by.size().reset_index(name='group_size')
 df = pd.merge(df, match_group_size, how='left', on=['matchId', 'groupId'], copy=False)
 del match_group_size; gc.collect()
match_group_max = match_group_by.max()
match_group_max_rank = match_group_max.groupby('matchId')[x_numeric_columns].rank(pct=True)
 df = pd.merge(df, match_group_max.reset_index() , on=['matchId', 'groupId'], how='left', suffixes=["", "_group_max"], copy=False)
 df = pd.merge(df, match_group_max_rank, on=['matchId', 'groupId'], how='left', suffixes=["", "_group_max_rank"], copy=False)
 del match_group_max, match_group_max_rank; gc.collect()
match_group_min = match_group_by.min()
match_group_min_rank = match_group_min.groupby('matchId')[x_numeric_columns].rank(pct=True)
 df = pd.merge(df, match_group_min.reset_index() , on=['matchId', 'groupId'], how='left', suffixes=["", "_group_min"], copy=False)
 df = pd.merge(df, match_group_min_rank, on=['matchId', 'groupId'], how='left', suffixes=["", "_group_min_rank"], copy=False)
 del match_group_min, match_group_min_rank; gc.collect()
df = optimize_memory(df)
 match_group_sum = match_group_by.sum()
match_group_sum.replace([np.inf, -np.inf], np.nan, inplace=True)
 match_group_sum.fillna(0, inplace=True)
 match_group_sum_rank = match_group_sum.groupby('matchId')[x_numeric_columns].rank(pct=True)
 df = pd.merge(df, match_group_sum.reset_index() , on=['matchId', 'groupId'], how='left', suffixes=["", "_group_sum"], copy=False)
 df = pd.merge(df, match_group_sum_rank, on=['matchId', 'groupId'], how='left', suffixes=["", "_group_sum_rank"], copy=False)
 del match_group_sum, match_group_sum_rank, match_group_by; gc.collect()
df = optimize_memory(df)
 print('Group matches done')
 match_by = df.groupby(['matchId'])[x_numeric_columns]
match_mean = match_by.mean().reset_index()
df = pd.merge(df, match_mean, on=['matchId'], how='left', suffixes=["", "_match_mean"], copy=False)
 del match_mean; gc.collect()
print('Matches mean done')
 match_median = match_by.median().reset_index()
df = pd.merge(df, match_median, on=['matchId'], how='left', suffixes=["", "_match_median"], copy=False)
 del match_median; gc.collect()
print('Matches median done')
 match_max = match_by.max().reset_index()
df = pd.merge(df, match_max, on=['matchId'], how='left', suffixes=["", "_match_max"], copy=False)
 del match_max; gc.collect()
print('Matches max done')
 match_min = match_by.min().reset_index()
df = pd.merge(df, match_min, on=['matchId'], how='left', suffixes=["", "_match_min"], copy=False)
 del match_min, match_by; gc.collect()
print('Matches min done')
 return df
def prepare_data(data, scaler=None):
data = add_features(data)
 print("Added more features")
 columns = [col for col in data.columns if col.split('_')[0] in x_numeric_columns] + ['group_size']
drop_columns = list(set(data.columns)- set(columns))
print("Columns to drop: {}".format(drop_columns))
match_type_cats = pd.get_dummies(data[x_categorical])
 data = pd.merge(data, match_type_cats, left_index=True, right_index=True, how='left', suffixes=["", ""], copy=False)
 del match_type_cats
data.drop(columns=drop_columns, inplace=True)
 gc.collect()
print('Start scaler')
 data_len = len(data)
 step = 50000
batches = np.arange(0, data_len, step)
 if scaler is None:
scaler = StandardScaler(copy=False)
 print('start scaler fitting')
 for batch in batches:
print("scaler fit batch {}".format(batch))
scaler.partial_fit(data.loc[batch:batch+step, columns].astype(np.float32))
gc.collect()
print('Start transform')
 for batch in batches:
print("scaler transform batch {}".format(batch))
data.loc[batch:batch+step, columns] = scaler.transform(data.loc[batch:batch+step, columns])
 gc.collect()
data = optimize_memory(data)
 return(data, scaler)
 def build_model(input_shape):
input_layer = Input(shape=input_shape)
 dense_1 = Dense(512, activation='relu' )(input_layer)
 dense_2 = Dense(256, activation='relu' )(dense_1)
 dense_3 = Dense(256, activation='relu' )(dense_2)
 output_layer = Dense(1, activation='sigmoid' )(dense_3)
 model = Model(inputs=input_layer, outputs=output_layer)
 model.compile(optimizer=keras.optimizers.Adam(lr=0.001), loss='mae')
 return model
def normalize_predictions(predicted, max_place):
predicted_place = max_place - predicted *(max_place - 1)
 predicted_place = predicted_place.round()
norm_predicted =(max_place - predicted_place)/(max_place - 1 + 0.0001)
 return norm_predicted
class LRSchedulerByLoss(Callback):
def __init__(self, decay=0.95, tolerans=5, verbose=0):
super(LRSchedulerByLoss, self ).__init__()
self.decay = decay
self.tolerans = tolerans
self.verbose = verbose
self.loss_not_decrease_epochs = 0
self.min_lost = float('inf')
 def on_train_begin(self, logs=None):
self.current_lr = float(K.get_value(self.model.optimizer.lr))
def on_epoch_end(self, epoch, logs={}):
current_loss = logs.get('loss')
 if self.min_lost - current_loss > 0.0001:
self.min_lost = current_loss
self.loss_not_decrease_epochs = 0
print('Improved')
 else:
self.loss_not_decrease_epochs += 1
print('Not improved')
 if self.loss_not_decrease_epochs > self.tolerans:
self.loss_not_decrease_epochs = self.loss_not_decrease_epochs // 2
self.current_lr = self.current_lr * self.decay
K.set_value(self.model.optimizer.lr, self.current_lr)
 if self.verbose > 0:
print('
Epoch %05d: LRSchedulerByLoss setting lr to %s.' %(epoch + 1, self.current_lr)) 
<train_model>
clf.predict(x_test )
train_x, scaler = prepare_data(source_data)
 input_shape =(train_x.shape[1],)
 print("Input shape {}".format(input_shape))
display(train_x.head())
 display(train_x.info())
 model = build_model(input_shape)
 gc.collect()
early_stopping = EarlyStopping(monitor='loss', min_delta=0.0001, patience=10, verbose=1, mode='auto', restore_best_weights=True)
 lr_scheduler_by_loss = LRSchedulerByLoss(verbose=1)
 model.fit(train_x.values, source_y, batch_size=batch_size, epochs=epochs, validation_split=validation_split, callbacks=[lr_scheduler_by_loss, early_stopping])
<prepare_x_and_y>
submission = pd.DataFrame(index=df_test.review_id)
 submission['is_good_rating'] = clf.predict(x_test )
<predict_on_test>
submission.reset_index().to_csv('submission.csv', index=False )