Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<choose_model_class>
init_notebook_mode(connected=True)
 %matplotlib inline
<install_modules>
env = twosigmanews.make_env()
!pip install -q efficientnet<set_options>
( market_train_df, news_train_df)= env.get_training_data()
print('TensorFlow version', tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE<define_variables>
def fix_time_col(in_mkt_df, in_news_df):
if "datetime" in str(in_mkt_df["time"].dtype):
in_mkt_df["time"] = in_mkt_df["time"].dt.date
in_news_df["time"] = in_news_df["time"].dt.date
def clean_assetName_col(in_mkt_df, in_news_df):
in_mkt_df["assetName"] = in_mkt_df["assetName"].str.lower()
in_mkt_df["assetName"] = in_mkt_df["assetName"].str.strip()
in_mkt_df["assetName"] = in_mkt_df["assetName"].str.replace("inc","")
 in_mkt_df["assetName"] = in_mkt_df["assetName"].str.replace("llc","")
 in_mkt_df["assetName"] = in_mkt_df["assetName"].str.replace("ltd","")
 in_mkt_df["assetName"] = in_mkt_df["assetName"].str.strip()
in_news_df["assetName"] = in_news_df["assetName"].str.lower()
in_news_df["assetName"] = in_news_df["assetName"].str.strip()
in_news_df["assetName"] = in_news_df["assetName"].str.replace("inc","")
 in_news_df["assetName"] = in_news_df["assetName"].str.replace("llc","")
 in_news_df["assetName"] = in_news_df["assetName"].str.replace("ltd","")
 in_news_df["assetName"] = in_news_df["assetName"].str.strip()
start_time = datetime.now()
print('Time now is', start_time)
 end_training_by_tdelta = timedelta(seconds=8400)
 this_run_file_prefix = start_time.strftime('%Y%m%d_%H%M_')
 print(this_run_file_prefix)
 IMAGE_SIZE = [224, 224]
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 MOREIMAGES_PATH_SELECT = {
192: '/tfrecords-jpeg-192x192',
224: '/tfrecords-jpeg-224x224',
331: '/tfrecords-jpeg-331x331',
512: '/tfrecords-jpeg-512x512'
}
MOREIMAGES_PATH = MOREIMAGES_PATH_SELECT[IMAGE_SIZE[0]]
IMAGENET_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/imagenet' + MOREIMAGES_PATH + '/*.tfrec')
 INATURELIST_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/inaturalist' + MOREIMAGES_PATH + '/*.tfrec')
 OPENIMAGE_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/openimage' + MOREIMAGES_PATH + '/*.tfrec')
 OXFORD_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/oxford_102' + MOREIMAGES_PATH + '/*.tfrec')
 TENSORFLOW_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/tf_flowers' + MOREIMAGES_PATH + '/*.tfrec')
 ADDITIONAL_TRAINING_FILENAMES = IMAGENET_FILES + INATURELIST_FILES + OPENIMAGE_FILES + OXFORD_FILES + TENSORFLOW_FILES
print('----')
 TRAINING_FILENAMES = TRAINING_FILENAMES + ADDITIONAL_TRAINING_FILENAMES
CLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle',
'snapdragon', "colt's foot", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william',
'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion',
'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy',
'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium',
'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily',
'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose']<choose_model_class>
def get_final_assetName(row):
min_mkt_assetNameSNo = row["min_mkt_assetNameSNo"]
max_mkt_assetNameSNo = row["max_mkt_assetNameSNo"]
min_assetNameSNo = row["min_assetNameSNo"]
max_assetNameSNo = row["max_assetNameSNo"]
if(( not math.isnan(min_mkt_assetNameSNo)) &(math.isnan(max_mkt_assetNameSNo))):
return min_assetNameSNo
elif(( not math.isnan(max_mkt_assetNameSNo)) &(math.isnan(min_mkt_assetNameSNo))):
return max_assetNameSNo
else:
return 0
def get_replace_assetName(row):
min_mkt_assetNameSNo = row["min_mkt_assetNameSNo"]
max_mkt_assetNameSNo = row["max_mkt_assetNameSNo"]
min_assetNameSNo = row["min_assetNameSNo"]
max_assetNameSNo = row["max_assetNameSNo"]
if(( math.isnan(min_mkt_assetNameSNo)) &(not math.isnan(max_mkt_assetNameSNo))):
return min_assetNameSNo
elif(( math.isnan(max_mkt_assetNameSNo)) &(not math.isnan(min_mkt_assetNameSNo))):
return max_assetNameSNo
else:
return 0
dic_assetName={}
dic_assetNameSNo={}
dic_assetCodes={}
def set_assetNameCodes_SNo(in_mkt_df, in_news_df,dic_assetName, dic_assetNameSNo, dic_assetCodes):
set_asset = set(in_news_df["assetName"].unique())| set(in_mkt_df["assetName"].unique())
 set_codes = set(in_news_df["assetCodes"].unique())
 list_asset = list(set_asset)
 list_codes = list(set_codes)
 if len(dic_assetName)== 0:
dic_assetName= {k:v for v,k in enumerate(list_asset)}
dic_assetNameSNo= {v:k for v,k in enumerate(list_asset)}
dic_assetCodes= {k:v for v,k in enumerate(list_codes)}
else:
set_asset_new = set(dic_assetName.keys())- set_asset
asset_max_SNo = len(dic_assetName)
 dic_assetName_new = {k:v+asset_max_SNo for v,k in enumerate(list(set_asset_new)) }
dic_assetNameSNo_new = {v+asset_max_SNo:k for v,k in enumerate(list(set_asset_new)) }
dic_assetName.update(dic_assetName_new)
 dic_assetNameSNo.update(dic_assetNameSNo_new)
 set_codes_new = set(dic_assetCodes.keys() -set_codes)
 codes_max_SNo = len(dic_assetCodes)
 dic_assetCodes_new = {k:v+codes_max_SNo for v,k in enumerate(list(set_codes_new)) }
dic_assetCodes.update(dic_assetCodes_new)
 in_news_df["assetNameSNo"] = in_news_df["assetName"].map(dic_assetName)
 in_mkt_df["assetNameSNo"] = in_mkt_df["assetName"].map(dic_assetName)
 in_news_df["assetCodesSNo"] = in_news_df["assetCodes"].map(dic_assetCodes)
 return dic_assetName, dic_assetNameSNo, dic_assetCodes
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = LR_START
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY = 0.80
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr = LR_START +(epoch *(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS)
 elif epoch <(LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS):
lr = LR_MAX
else:
lr = LR_MIN +(LR_MAX - LR_MIN)* LR_EXP_DECAY **(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)
 return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 rng = [i for i in range(20)]
y = [lrfn(x)for x in rng]
print(y )<categorify>
def replace_assetName(in_mkt_df, in_news_df):
in_news_df["num_rows"] = 1
assetName_df = in_news_df[["assetCodesSNo","assetNameSNo","num_rows"]].groupby(\
["assetCodesSNo","assetNameSNo"], as_index=False ).count()
assetCode_df = assetName_df[["assetCodesSNo", \
"assetNameSNo","num_rows"]].groupby(["assetCodesSNo"], \
as_index=False ).agg({"assetNameSNo": ["max","min"], "num_rows": "sum"})
 assetCode_df.columns = ["assetCodesSNo", "max_assetNameSNo", "min_assetNameSNo", "count_num_rows"]
assetCode_df = assetCode_df[assetCode_df["max_assetNameSNo"] != assetCode_df["min_assetNameSNo"]]
if assetCode_df.shape[0] > 0:
print(assetCode_df.shape)
 assetCode_df["max_assetName"] = assetCode_df["max_assetNameSNo"].map(dic_assetNameSNo)
 assetCode_df["min_assetName"] = assetCode_df["min_assetNameSNo"].map(dic_assetNameSNo)
 list_mkt_asset = list(in_mkt_df["assetName"].unique())
 dic_mkt_assetName= {k:v for v,k in enumerate(list_mkt_asset)}
assetCode_df["min_mkt_assetNameSNo"] = assetCode_df["min_assetName"].map(dic_mkt_assetName)
 assetCode_df["max_mkt_assetNameSNo"] = assetCode_df["max_assetName"].map(dic_mkt_assetName)
 assetCode_df["final_assetNameSNo"] = assetCode_df.apply(get_final_assetName, axis=1)
 assetCode_df["replace_assetNameSNo"] = assetCode_df.apply(get_replace_assetName, axis=1)
 assetCode_df["final_assetName"] = assetCode_df["final_assetNameSNo"].map(dic_assetNameSNo)
 df_temp = pd.merge(in_news_df.reset_index() [["index","assetCodesSNo","assetNameSNo"]], \
assetCode_df[["replace_assetNameSNo", "final_assetNameSNo","assetCodesSNo","final_assetName"]], \
how="inner", left_on=["assetCodesSNo","assetNameSNo"], right_on=["assetCodesSNo","replace_assetNameSNo"], left_index=True ).set_index("index")
 in_news_df["new_assetNameSNo"] = -1
in_news_df["new_assetName"] = ""
in_news_df.loc[df_temp.index, "assetNameSNo"] = list(df_temp["final_assetNameSNo"])
 in_news_df.loc[df_temp.index, "assetName"] = list(df_temp["final_assetName"] )
np.set_printoptions(threshold=15, linewidth=80)
 def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_flower(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else CLASSES[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()
def display_confusion_matrix(cmat, score, precision, recall):
plt.figure(figsize=(15,15))
ax = plt.gca()
ax.matshow(cmat, cmap='Reds')
 ax.set_xticks(range(len(CLASSES)))
 ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_xticklabels() , rotation=45, ha="left", rotation_mode="anchor")
 ax.set_yticks(range(len(CLASSES)))
 ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_yticklabels() , rotation=45, ha="right", rotation_mode="anchor")
 titlestring = ""
if score is not None:
titlestring += 'f1 = {:.3f} '.format(score)
 if precision is not None:
titlestring += '
precision = {:.3f} '.format(precision)
 if recall is not None:
titlestring += '
recall = {:.3f} '.format(recall)
 if len(titlestring)> 0:
ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'
plt.show()
def display_training_curves(training, validation, title, subplot):
if subplot%10==1:
plt.subplots(figsize=(10,10), facecolor='
plt.tight_layout()
ax = plt.subplot(subplot)
 ax.set_facecolor('
ax.plot(training)
 ax.plot(validation)
 ax.set_title('model '+ title)
 ax.set_ylabel(title)
 ax.set_xlabel('epoch')
 ax.legend(['train', 'valid.'] )<categorify>
dic_rng = {}
dic_min = {}
def scale_col(in_news_df, col):
global dic_rng
global dic_min
if col not in dic_rng.keys() :
col_max = in_news_df[col].max()
col_min = in_news_df[col].min()
col_rng = col_max - col_min
dic_rng[col] = col_rng
dic_min[col] = col_min
else:
col_min = dic_min[col]
col_rng = dic_rng[col]
in_news_df[col] =(in_news_df[col]-col_min)/col_rng
if col_min < 0:
in_news_df[col] = 2*in_news_df[col] - 1
print(col, in_news_df[col].min() , in_news_df[col].max())
 def drop_extra_news_cols(in_news_df):
news_drop_cols = ["sourceTimestamp","firstCreated","sourceId","headline","urgency","takeSequence","provider","audiences","bodySize",\
"headlineTag","sentenceCount","wordCount","firstMentionSentence","sentimentClass","sentimentWordCount"]
news_drop_cols = [col for col in news_drop_cols if col in in_news_df.columns ]
in_news_df.drop(news_drop_cols, axis=1, inplace=True)
 def drop_irrelevant_news(in_news_df):
irrelevant_news = in_news_df[in_news_df["relevance"] < 0.3].index
print("Num low relevance news:", len(irrelevant_news))
in_news_df.drop(irrelevant_news, axis=0, inplace=True)
 del irrelevant_news
gc.collect()
def drop_neutral_news(in_news_df):
in_news_df["netSentimentPositive"] =(in_news_df["sentimentPositive"] - in_news_df["sentimentNegative"])
 neutral_news = in_news_df[in_news_df["netSentimentPositive"].abs() < 0.05].index
in_news_df.drop(neutral_news, axis=0, inplace=True)
 in_news_df["netSentimentPositive"] = in_news_df["netSentimentPositive"] *(1 - in_news_df["sentimentNeutral"])
 print("Num neutral news:", len(neutral_news))
del neutral_news
gc.collect()
def gen_netnovelty_col(in_news_df):
arr_novelty = np.array(in_news_df[["noveltyCount12H","noveltyCount24H","noveltyCount3D","noveltyCount5D","noveltyCount7D"]])
 in_news_df["inv_netNovelty"] = list(1/(np.argmax(arr_novelty, axis=1)+2))
col = "inv_netNovelty"
scale_col(in_news_df, col)
 setSector = {"TECH","ENER","BMAT","INDS","CYCS","NCYC","FINS","HECA","TECH","TCOM","UTIL"}
def get_sector_name(in_news_df):
global setSector
sectorList = {"ENER":"Energy", "BMAT":"Basic Materials", "INDS":"Industrials", "CYCS":"Cyclical Consumer Goods & Services", \
"NCYC":"Non Cyclical Consumer Goods & Services", "FINS": "Financials", "HECA":"Healthcare","TECH":"Technology", \
"TCOM":"Telecommunication Services", "UTIL":"Utilities"}
in_news_df["sector"] = in_news_df["subjects"].apply(lambda x: str(eval(x)& setSector))
for col in setSector:
print(col)
 in_news_df[col] = in_news_df["sector"].apply(lambda x: col in eval(x))
lst_sector = list(setSector)
 dic_sector = {v: k for v,k in enumerate(list(setSector)) }
lst_sector.append("assetNameSNo")
 asset_df = in_news_df[lst_sector].groupby(["assetNameSNo"] ).sum()
asset_df1 = asset_df.sum(axis=1)
 asset_unknown_sector = list(asset_df1[asset_df1==0].index)
 print("Could not find sector of ", len(asset_unknown_sector), " assets")
 asset_df.drop(asset_unknown_sector, axis=0, inplace=True)
 asset_df["sectorKey"] = asset_df.values.argmax(axis=1)
 asset_df["sectorName"] = asset_df["sectorKey"].map(dic_sector)
 print(asset_df.head(1))
asset_df.reset_index(inplace=True)
 merged_df = pd.merge(in_news_df.reset_index() [["assetNameSNo","index"]], asset_df[["assetNameSNo","sectorName"]], \
left_on= ["assetNameSNo"], right_on= ["assetNameSNo"], how="left" ).set_index("index")
 in_news_df["sectorName"] = ""
in_news_df.loc[merged_df.index,["sectorName"]] = merged_df.loc[merged_df.index,["sectorName"]]
return asset_df[["assetNameSNo","sectorName"]]
def merge_mkt_news(in_mkt_df, in_news_df):
in_mkt_df["mkt_cap"] =(( in_mkt_df["close"] + in_mkt_df["open"])/2)* in_mkt_df["volume"]
in_mkt_df_grouped = in_mkt_df[["assetNameSNo","time", "close","returnsClosePrevRaw10","returnsClosePrevMktres10",\
"returnsOpenNextMktres10","mkt_cap"]].groupby(["assetNameSNo","time"], as_index=False ).sum()
merged_df = pd.merge(in_news_df.reset_index() [["index","assetNameSNo","time"]], in_mkt_df_grouped, left_on= ["assetNameSNo","time"], \
right_on= ["assetNameSNo","time"], how="inner" ).set_index("index")
 in_news_df["close"] = 0
in_news_df["returnsClosePrevRaw10"] = 0
in_news_df["returnsClosePrevMktres10"] = 0
in_news_df["returnsOpenNextMktres10"] = 0
in_news_df["mkt_cap"] = 0
in_news_df.loc[merged_df.index, ["close","returnsClosePrevRaw10","returnsClosePrevMktres10","returnsOpenNextMktres10","mkt_cap"]] = \
merged_df.loc[merged_df.index, ["close","returnsClosePrevRaw10","returnsClosePrevMktres10","returnsOpenNextMktres10","mkt_cap"]]
in_news_df["weighted_SenSR"] = in_news_df["SenSR"] * in_news_df["mkt_cap"]
def get_smooth_asset_sentiment(in_mkt_df, in_news_df):
col="SenSR"
asset_col = "asset_SenSR"
rolling_col = "rolling_" + asset_col
smooth_col = "smooth_" + asset_col
in_news_df[asset_col] = in_news_df[col]
in_news_df[rolling_col] = 0
in_news_df[smooth_col] = 0
lst_data = []
for key, val_df in in_news_df.groupby(["assetNameSNo"]):
print("Generating rolled and smooth col for asset:" + str(key)+ ":" + dic_assetNameSNo[key])
 val_df.set_index(["time"], inplace=True)
 val_df[rolling_col] = val_df[asset_col].rolling(window=7, min_periods=1 ).mean()
val_df.dropna(axis=0, inplace=True)
 y = list(val_df[rolling_col].values)
 x = range(val_df.shape[0])
 smooth = lowess(y,x, 0.02)
 val_df[smooth_col] = list(smooth[:,1])
 del smooth, x, y
lst_data.append(val_df[["assetNameSNo",asset_col,rolling_col,smooth_col]].reset_index())
 asset_df = pd.concat(lst_data, axis=0)
 merged_df = pd.merge(in_news_df.reset_index() [["index","assetNameSNo","time"]], asset_df, how="left", \
left_on=["assetNameSNo", "time"], right_on=["assetNameSNo","time"] ).set_index("index")
 in_news_df.loc[merged_df.index, [asset_col,rolling_col,smooth_col]] = merged_df.loc[merged_df.index, [asset_col,rolling_col,smooth_col]]
return merged_df
def get_smooth_sector_sentiment(in_mkt_df, in_news_df):
col = "weighted_SenSR"
sector_col = "sector_SenSR"
rolling_col = "rolling_" + sector_col
smooth_col = "smooth_" + sector_col
in_news_df[sector_col] = 0
in_news_df[rolling_col] = 0
in_news_df[smooth_col] = 0
lst_data = []
for key, val_df in in_news_df[[col,"time","sectorName","mkt_cap"]].groupby(["sectorName"]):
print("Generating rolled and smooth col for sector:" + str(key))
val_df = val_df.groupby(["time","sectorName"], as_index=False ).sum()
val_df[sector_col] = val_df[col]/val_df["mkt_cap"]
val_df.set_index(["time"], inplace=True)
 val_df[rolling_col] = val_df[sector_col].rolling(window=7, min_periods=1 ).mean()
val_df.dropna(axis=0, inplace=True)
 y = list(val_df[rolling_col].values)
 x = range(val_df.shape[0])
 smooth = lowess(y,x, 0.02)
 val_df[smooth_col] = list(smooth[:,1])
 del smooth, x, y
lst_data.append(val_df[[sector_col,rolling_col,smooth_col, "sectorName"]].reset_index())
 sector_df = pd.concat(lst_data, axis=0)
 merged_df = pd.merge(in_news_df.reset_index() [["index","sectorName","time"]], sector_df, how="left", \
left_on=["sectorName", "time"], right_on=["sectorName","time"] ).set_index("index")
 in_news_df.loc[merged_df.index, [sector_col,rolling_col,smooth_col]] = merged_df.loc[merged_df.index, [sector_col,rolling_col,smooth_col]]
return merged_df
def get_smooth_overall_sentiment(in_mkt_df, in_news_df):
col = "weighted_SenSR"
all_col = "all_SenSR"
rolling_col = "rolling_" + all_col
smooth_col = "smooth_" + all_col
in_news_df[all_col] = 0
in_news_df[rolling_col] = 0
in_news_df[smooth_col] = 0
lst_data = []
val_df = in_news_df[[col,"time","mkt_cap"]].groupby(["time"] ).sum()
val_df[all_col] = val_df[col]/val_df["mkt_cap"]
val_df[rolling_col] = val_df[all_col].rolling(window=7, min_periods=1 ).mean()
val_df.dropna(axis=0, inplace=True)
 y = list(val_df[rolling_col].values)
 x = range(val_df.shape[0])
 smooth = lowess(y,x, 0.02)
 val_df[smooth_col] = list(smooth[:,1])
 del smooth, x, y
lst_data.append(val_df[[all_col,rolling_col,smooth_col]].reset_index())
 sector_df = pd.concat(lst_data, axis=0)
 merged_df = pd.merge(in_news_df.reset_index() [["index","time"]], sector_df, how="left", \
left_on=["time"], right_on=["time"] ).set_index("index")
 in_news_df.loc[merged_df.index, [all_col,rolling_col,smooth_col]] = merged_df.loc[merged_df.index, [all_col, rolling_col,smooth_col]]
return merged_df
def gen_news_features(in_mkt_df, in_news_df):
global setSector
drop_extra_news_cols(in_news_df)
 drop_irrelevant_news(in_news_df)
 drop_neutral_news(in_news_df)
 gen_netnovelty_col(in_news_df)
 sector_name_df = get_sector_name(in_news_df)
 in_news_df.drop("subjects", axis=1, inplace=True)
 in_news_df.drop(["noveltyCount12H","noveltyCount24H","noveltyCount3D","noveltyCount5D","noveltyCount7D"], axis=1, inplace=True)
 in_news_df.drop(["volumeCounts12H","volumeCounts24H","volumeCounts3D","volumeCounts5D","volumeCounts7D"], axis=1, inplace=True)
 in_news_df["SenSR"] = in_news_df["netSentimentPositive"] * in_news_df["relevance"] *(in_news_df["inv_netNovelty"])
 in_news_df = in_news_df.groupby(["assetName","sectorName","time","assetNameSNo"], as_index=False ).sum()
merge_mkt_news(in_mkt_df, in_news_df)
 asset_df = get_smooth_asset_sentiment(in_mkt_df,in_news_df)
 sector_df = get_smooth_sector_sentiment(in_mkt_df, in_news_df)
 all_df = get_smooth_overall_sentiment(in_mkt_df, in_news_df)
 return sector_name_df, asset_df, sector_df, all_df, in_news_df
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels = 3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'class': tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'id': tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_saturation(image, 0, 2)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled = True)
 dataset = dataset.map(data_augment, num_parallel_calls = AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered = False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled = True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered = False):
dataset = load_dataset(TEST_FILENAMES, labeled = False, ordered = ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<define_variables>
fix_time_col(market_train_df, news_train_df)
 clean_assetName_col(market_train_df, news_train_df)
 dic_assetName, dic_assetNameSNo, dic_assetCodes = set_assetNameCodes_SNo(market_train_df, news_train_df,dic_assetName, dic_assetNameSNo, dic_assetCodes)
 replace_assetName(market_train_df, news_train_df)
 sector_name_df, asset_df, sector_df, all_df, news_feature_df = gen_news_features(market_train_df, news_train_df)
training_dataset = get_training_dataset()
training_dataset = training_dataset.unbatch().batch(20)
 train_batch = iter(training_dataset)
<create_dataframe>
drop_cols = list(set(["companyCount","marketCommentary","assetCodesSNo","num_rows"])| setSector)
 news_feature_df.drop(drop_cols, axis=1, inplace=True)
 news_feature_df.tail(1)
test_dataset = get_test_dataset()
test_dataset = test_dataset.unbatch().batch(20)
 test_batch = iter(test_dataset)
<choose_model_class>
drop_cols = ["sentimentNegative","sentimentPositive","sentimentNeutral"]
news_feature_df.drop(drop_cols, axis=1, inplace=True )
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=20, restore_best_weights = True )<choose_model_class>
apple_df = news_feature_df[news_feature_df["assetName"]=="apple"]
apple_df["target"] = apple_df["returnsOpenNextMktres10"].apply(lambda x: 1 if x>0 else 2 if x<0 else 3)
 apple_df["asset_sector_SenSR"] = apple_df["smooth_asset_SenSR"] * apple_df["smooth_sector_SenSR"]
apple_df["asset_sector_all_SenSR"] = apple_df["smooth_asset_SenSR"] * apple_df["smooth_sector_SenSR"] * apple_df["smooth_all_SenSR"]
col_list1 = ["returnsClosePrevMktres10","returnsOpenNextMktres10",\
"asset_SenSR","rolling_asset_SenSR","smooth_asset_SenSR"]
col_list2 = ["returnsClosePrevMktres10","returnsOpenNextMktres10",\
"sector_SenSR","rolling_sector_SenSR","smooth_sector_SenSR"]
col_list3 = ["returnsClosePrevMktres10","returnsOpenNextMktres10",\
"asset_sector_SenSR","asset_sector_all_SenSR"]
for col in list(set(col_list1)|set(col_list2)|set(col_list3)) :
scale_col(apple_df, col)
 scale_col(apple_df, "smooth_all_SenSR")