Kaggle competition: Kannada MNIST
market_train = process_date(market_train)
 market_train = process_ma(market_train )<merge>
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.preprocessing.image import ImageDataGenerator
df = market_train.merge(news_agg, how='left', on=['assetCode', 'date'])
 del market_train, news_agg
gc.collect()
df.head(3 )<data_type_conversions>
train_data = pd.read_csv(".. /input/Kannada-MNIST/train.csv")
 test_data = pd.read_csv(".. /input/Kannada-MNIST/test.csv")
 print(train_data.shape,test_data.shape )
for column in df.select_dtypes(include='float64' ).columns:
df[column] = df[column].astype('float16')
 for column in df.select_dtypes(include='int64' ).columns:
df[column] = df[column].astype('int16')
 df.dtypes<compute_test_metric>
X = train_data.drop(["label"],axis = 1 ).values
Y = train_data["label"].values
def custom_metric(date, pred_proba, num_target, universe):
y = pred_proba*2 - 1
r = num_target.clip(-1,1)
 x = y * r * universe
result = pd.DataFrame({'day' : date, 'x' : x})
 x_t = result.groupby('day' ).sum().values
return np.mean(x_t)/ np.std(x_t )<data_type_conversions>
X = X.reshape([60000,28,28,1])
 Y = Y.reshape([60000,1] )
date = df.date
num_target = df.returnsOpenNextMktres10.astype('float32')
 bin_target =(df.returnsOpenNextMktres10 >= 0 ).astype('int8')
 universe = df.universe.astype('int8')
 df.drop(['returnsOpenNextMktres10', 'date', 'universe', 'assetCode', 'assetName', 'time'],
axis=1, inplace=True)
 df = df.astype('float32')
 gc.collect()<split>
Y = to_categorical(Y, num_classes = 10 )
train_index, test_index = train_test_split(df.index.values, test_size=0.1 )<init_hyperparams>
x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size = 0.1, random_state = 14 )
best_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
'learning_rate': 0.05,
'num_leaves': 63,
'n_estimators': 1000,
'min_child_samples': 20,
'colsample_bytree': 1,
'subsample': 1.0,
'reg_alpha': 0.1,
'reg_lambda': 0.8,
}<create_dataframe>
x_train = x_train/255
x_test = x_test/255
lgb_train = lgb.Dataset(df.iloc[train_index], bin_target.iloc[train_index])
 lgb_eval = lgb.Dataset(df.iloc[test_index], bin_target.iloc[test_index], reference=lgb_train )<train_model>
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(64,(3,3), padding = 'same', activation='relu', input_shape=(28, 28, 1)) ,
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Conv2D(64,(3,3), padding = 'same', activation='relu'),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Conv2D(128,(3,3), padding = 'same', activation='relu'),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Conv2D(128,(3,3), padding = 'same', activation='relu'),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(10, activation='softmax')
 ] )
clf = lgb.train(
 best_params,
lgb_train,
valid_sets=lgb_eval,
early_stopping_rounds=5
)<predict_on_test>
optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999)
 model.compile(optimizer = optimizer,
loss = 'categorical_crossentropy',
metrics = ['accuracy'] )
pred_y = clf.predict(df.iloc[test_index])
 pred_y[pred_y >= 0.5] = 1
pred_y[pred_y < 0.5] = 0
print(( pred_y == bin_target.iloc[test_index] ).sum() / len(pred_y))<feature_engineering>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.6,
min_lr=0.00001 )
test_df_columns = ['time', 'assetCode', 'assetName', 'volume', 'close', 'open',
'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',
'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',
'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10']
base_df = market_train_df[market_train_df['time'] >= '2016-10-01']
base_df = base_df[test_df_columns]
base_df['id'] = -1
base_df.shape<categorify>
batch_size = 128
epochs = 50
def calc_target(df):
df['open_next10'] = df.groupby('assetCode')['open'].shift(-10)
 df['return_next10'] = df['open_next10'] / df['open'] - 1.0
df['target'] =(df.return_next10 >= 0 ).astype('int8')
 return df<prepare_output>
train_datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range = 0.1,
horizontal_flip=False,
vertical_flip=False 
)
 train_datagen.fit(x_train )
target_test_df = calc_target(market_train_df)
 target_test_df.head()<train_model>
history = model.fit(
 train_datagen.flow(x_train,y_train,batch_size = batch_size),
validation_data =(x_test,y_test),
batch_size = batch_size,
steps_per_epoch = x_train.shape[0]//batch_size,
epochs = epochs,
verbose = 1,
callbacks=[learning_rate_reduction]
)
def online_train(model, df, train_day_id, feats):
best_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
'learning_rate': 0.05,
'num_leaves': 63,
'n_estimators': 1000,
'min_child_samples': 20,
'colsample_bytree': 1,
'subsample': 1.0,
'reg_alpha': 0.1,
'reg_lambda': 0.8,
}
train_df = calc_target(df)
 train_df = train_df[train_df['id'] == train_day_id]
train_df.reset_index(drop=True)
 target_df = train_df[['target']]
train_df = train_df[feats]
model.refit(train_df, target_df['target'])
 return model

<categorify>
model.evaluate(x_test,y_test )
def write_submission(model, env):
days = env.get_prediction_days()
day_id = 0
market_obs_df_append = None
online_train_df_append = None
for(market_obs_df, news_obs_df, predictions_template_df)in days:
news_obs_df = preprocess_news(news_obs_df)
 index_df = unstack_asset_codes(news_obs_df)
 news_unstack = merge_news_on_index(news_obs_df, index_df)
 news_obs_agg = group_news(news_unstack)
 market_obs_df['id'] = day_id
if market_obs_df_append is None:
market_obs_df_append = base_df
market_obs_df_append = pd.concat([market_obs_df_append,market_obs_df],
ignore_index=True,
sort=False)
 market_obs_process = process_date(market_obs_df_append)
 market_obs_process = process_ma(market_obs_process)
 market_obs_df = market_obs_process[market_obs_process['id']==day_id]
obs_df = market_obs_df.merge(news_obs_agg, how='left', on=['assetCode', 'date'])
 del market_obs_df, news_obs_agg, news_obs_df, news_unstack, index_df
gc.collect()
obs_df = obs_df[obs_df.assetCode.isin(predictions_template_df.assetCode)]
feats = [c for c in obs_df.columns if c not in ['date', 'assetCode', 'assetName', 'time', 'id']]
if online_train_df_append is None:
online_train_df_append = obs_df
else:
online_train_df_append = pd.concat([online_train_df_append, obs_df],
ignore_index=True,
sort=False)
 if day_id >= 10:
model = online_train(model, online_train_df_append, day_id - 10, feats)
 preds = model.predict(obs_df[feats])* 2 - 1
sub = pd.DataFrame({'assetCode': obs_df['assetCode'], 'confidence': preds})
 predictions_template_df = predictions_template_df.merge(sub, how='left' ).drop(
 'confidenceValue', axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 if day_id == 59:
market_obs_df_append.drop(
 market_obs_df_append.index[market_obs_df_append['id']==-1],
inplace=True)
 elif day_id >= 60:
market_obs_df_append.drop(
 market_obs_df_append.index[market_obs_df_append['id']==day_id-60],
inplace=True)
 day_id += 1
del obs_df, predictions_template_df, preds, sub
gc.collect()
env.write_submission_file()
print('day_count',day_id )<save_to_csv>
y_pred = model.predict(x_test)
 X_new = x_test.reshape(x_test.shape[0], 28, 28)
 fig, axis = plt.subplots(4, 4, figsize=(12, 14))
for i, ax in enumerate(axis.flat):
ax.imshow(X_new[i], cmap='binary')
 ax.set(title = f"Real Number is {y_test[i].argmax() }
Predict Number is {y_pred[i].argmax() }");
write_submission(clf, env )<feature_engineering>
raw_test = test_data
sample = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv' )
if 'env' not in globals() :
env = twosigmanews.make_env() 
(market_train_df, news_train_df)= env.get_training_data()
market_train_df['time'] = market_train_df['time'].dt.date
market_train_df = market_train_df.loc[market_train_df['time']>=date(2010, 1, 1)]
market_train, news_train = market_train_df.copy() , news_train_df.copy()<merge>
sample_sub=sample
raw_test_id=raw_test.id
raw_test=raw_test.drop("id",axis="columns")
 raw_test=raw_test / 255
test=raw_test.values.reshape(-1,28,28,1)
 test.shape
<categorify><EOS> 
sub=model.predict(test)
 sub=np.argmax(sub,axis=1)
 sample_sub['label']=sub
sample_sub.to_csv('submission.csv',index=False )