Repository: ranaroussi/yfinance

Issue:
the latest date OLHC data appears NaN during dividend and trading hours
so I ran this command, 

yf.Ticker('ALB').history(period='1d')

its supposed to look like this 

This is from yahoo finance at the time, during trading hours.



Date | Open | High | Low | Close* | Adj Close** | Volume
-- | -- | -- | -- | -- | -- | --
Mar 17, 2022 | 187.84 | 196.09 | 187.84 | 195.21 | 195.21 | 403,080
Mar 17, 2022 | 0.395 Dividend
Mar 16, 2022 | 185.87 | 193.98 | 185.76 | 191.25 | 190.85 | 890,700
Mar 15, 2022 | 175.36 | 182.70 | 172.78 | 181.95 | 181.57 | 834,400

But this is what I got.

Date | Open | High | Low | Close | Volume | Dividends | Stock Splits
-- | -- | -- | -- | -- | -- | -- | --
2022-03-17 | NaN | NaN | NaN | NaN | NaN | 0.395 | 0

It only showed me the latest empty dividend data, not the prices above it. even if I run this

yf.Ticker('ALB').history(),

the latest date shows me only the blank dividend data when it should be showing me 

Date | Open | High | Low | Close* | Adj Close** | Volume
-- | -- | -- | -- | -- | -- | --
Mar 17, 2022 | 187.84 | 196.09 | 187.84 | 195.21 | 195.21 | 403,080


according to the yahoo finance data

this seems to happen because there was a dividend today for this stock, on the yahoo finance website it shows


Date | Open | High | Low | Close* | Adj Close** | Volume
-- | -- | -- | -- | -- | -- | --
Mar 17, 2022 | 187.84 | 196.09 | 187.84 | 195.21 | 195.21 | 403,080
Mar 17, 2022 | 0.395 Dividend
Mar 16, 2022 | 185.87 | 193.98 | 185.76 | 191.25 | 190.85 | 890,700
Mar 15, 2022 | 175.36 | 182.70 | 172.78 | 181.95 | 181.57 | 834,400


but in the Python API, it only shows up till the dividend split and gives me Nans in the other columns

if I use pandas data reader,

import pandas_datareader as pdr
pdr.get_data_yahoo('ALB')

I get the proper data like this,



  | High | Low | Open | Close | Volume | Adj Close
-- | -- | -- | -- | -- | -- | --

106.190002 | 105.120003 | 106.040001 | 106.070000 | 1032400.0 | 98.987755
107.559998 | 104.070000 | 106.070000 | 104.250000 | 1249500.0 | 97.289299
105.470001 | 103.300003 | 104.599998 | 105.220001 | 984800.0 | 98.194519
106.389999 | 104.849998 | 105.180000 | 105.150002 | 690000.0 | 98.129196
106.180000 | 104.029999 | 105.540001 | 104.449997 | 527400.0 | 97.475929
... | ... | ... | ... | ... | ...
191.449997 | 184.639999 | 188.770004 | 184.759995 | 769600.0 | 184.378403
186.169998 | 175.259995 | 184.869995 | 176.289993 | 973200.0 | 175.925888
182.699997 | 172.779999 | 175.360001 | 181.949997 | 834400.0 | 181.574203
193.979996 | 185.759995 | 185.869995 | 191.250000 | 890700.0 | 190.854996
196.360001 | 187.839996 | 187.839996 | 196.205002 | 487406.0 | 196.205002



Discussion:
It is currently happening again with the ticker WPM and PM today.

I have also observed, suffered, from this feature. It seems to have started when I updated to the latest version of yfinance.

Resolving patch:
diff --git a/yfinance/base.py b/yfinance/base.py
--- a/yfinance/base.py
+++ b/yfinance/base.py
@@ -288,13 +288,32 @@ def history(self, period="1mo", interval="1d",
 
         tz_exchange = data["chart"]["result"][0]["meta"]["exchangeTimezoneName"]
 
-        # combine
-        df = _pd.concat([quotes, dividends, splits], axis=1, sort=True)
-        df["Dividends"].fillna(0, inplace=True)
-        df["Stock Splits"].fillna(0, inplace=True)
+        # prepare index for combine:
+        quotes.index = quotes.index.tz_localize("UTC").tz_convert(tz_exchange)
+        splits.index = splits.index.tz_localize("UTC").tz_convert(tz_exchange)
+        dividends.index = dividends.index.tz_localize("UTC").tz_convert(tz_exchange)
+        if params["interval"] in ["1d","1w","1wk","1mo","3mo"]:
+            # Converting datetime->date should improve merge performance
+            quotes.index = _pd.to_datetime(quotes.index.date)
+            splits.index = _pd.to_datetime(splits.index.date)
+            dividends.index = _pd.to_datetime(dividends.index.date)
 
-        # index eod/intraday
-        df.index = df.index.tz_localize("UTC").tz_convert(tz_exchange)
+        # combine
+        df = quotes
+        if actions:
+            df = df.sort_index()
+            if dividends.shape[0] > 0:
+                df = utils.safe_merge_dfs(df, dividends, interval)
+            if "Dividends" in df.columns:
+                df.loc[df["Dividends"].isna(),"Dividends"] = 0
+            else:
+                df["Dividends"] = 0.0
+            if splits.shape[0] > 0:
+                df = utils.safe_merge_dfs(df, splits, interval)
+            if "Stock Splits" in df.columns:
+                df.loc[df["Stock Splits"].isna(),"Stock Splits"] = 0
+            else:
+                df["Stock Splits"] = 0.0
 
         df = utils.fix_Yahoo_dst_issue(df, params["interval"])
             
@@ -303,7 +322,6 @@ def history(self, period="1mo", interval="1d",
         elif params["interval"] == "1h":
             pass
         else:
-            df.index = _pd.to_datetime(df.index.date).tz_localize(tz_exchange)
             df.index.name = "Date"
 
         # duplicates and missing rows cleanup
@@ -312,9 +330,6 @@ def history(self, period="1mo", interval="1d",
 
         self._history = df.copy()
 
-        if not actions:
-            df.drop(columns=["Dividends", "Stock Splits"], inplace=True)
-
         return df
 
     # ------------------------
diff --git a/yfinance/utils.py b/yfinance/utils.py
--- a/yfinance/utils.py
+++ b/yfinance/utils.py
@@ -247,11 +247,134 @@ def parse_actions(data):
             splits.sort_index(inplace=True)
             splits["Stock Splits"] = splits["numerator"] / \
                 splits["denominator"]
-            splits = splits["Stock Splits"]
+            splits = splits[["Stock Splits"]]
 
     return dividends, splits
 
 
+def safe_merge_dfs(df_main, df_sub, interval):
+    # Carefully merge 'df_sub' onto 'df_main'
+    # If naive merge fails, try again with reindexing df_sub:
+    # 1) if interval is weekly or monthly, then try with index set to start of week/month
+    # 2) if still failing then manually search through df_main.index to reindex df_sub
+
+    if df_sub.shape[0] == 0:
+        raise Exception("No data to merge")
+    
+    df_sub_backup = df_sub.copy()
+    data_cols = [c for c in df_sub.columns if not c in df_main]
+    if len(data_cols) > 1:
+        raise Exception("Expected 1 data col")
+    data_col = data_cols[0]
+
+    def _reindex_events(df, new_index, data_col_name):
+        if len(new_index) == len(set(new_index)):
+            # No duplicates, easy
+            df.index = new_index
+            return df
+
+        df["_NewIndex"] = new_index
+        # Duplicates present within periods but can aggregate
+        if data_col_name == "Dividends":
+            # Add
+            df = df.groupby("_NewIndex").sum()
+            df.index.name = None
+        elif data_col_name == "Stock Splits":
+            # Product
+            df = df.groupby("_NewIndex").prod()
+            df.index.name = None
+        else:
+            raise Exception("New index contains duplicates but unsure how to aggregate for '{}'".format(data_col_name))
+        if "_NewIndex" in df.columns:
+            df = df.drop("_NewIndex",axis=1)
+        return df
+
+    # Discard last row in 'df_sub' if significantly after last row in df_main.
+    # Size of difference depends on interval.
+    df_sub = df_sub[df_sub.index >= df_main.index[0]]
+    df_sub_last_dt = df_sub.index[-1]
+    df_main_last_dt = df_main.index[-1]
+    if df_sub_last_dt > df_main_last_dt:
+        if interval == "1mo" and df_sub_last_dt.month != df_main_last_dt.month:
+            df_sub = df_sub.drop(df_sub.index[-1])
+        elif interval in ["1wk","5d"] and df_sub_last_dt.week != df_main_last_dt.week:
+            df_sub = df_sub.drop(df_sub.index[-1])
+        elif interval == "1d" and df_sub_last_dt.date() > df_main_last_dt.date():
+            df_sub = df_sub.drop(df_sub.index[-1])
+        elif (interval.endswith('h') or interval.endswith('m')) and (df_sub_last_dt.date() > df_main_last_dt.date()):
+            df_sub = df_sub.drop(df_sub.index[-1])
+        if df_sub.shape[0] == 0:
+            # raise Exception("No data to merge after pruning out-of-range")
+            return df_main
+
+    df = df_main.join(df_sub)
+
+    f_na = df[data_col].isna()
+    data_lost = sum(~f_na) < df_sub.shape[0]
+    if not data_lost:
+        return df
+    # Lost data during join()
+    if interval in ["1wk","1mo","3mo"]:
+        # Backdate all df_sub.index dates to start of week/month
+        if interval == "1wk":
+            new_index = _pd.PeriodIndex(df_sub.index, freq='W').to_timestamp()
+        elif interval == "1mo":
+            new_index = _pd.PeriodIndex(df_sub.index, freq='M').to_timestamp()
+        elif interval == "3mo":
+            new_index = _pd.PeriodIndex(df_sub.index, freq='Q').to_timestamp()
+        df_sub = _reindex_events(df_sub, new_index, data_col)
+        df = df_main.join(df_sub)
+
+    f_na = df[data_col].isna()
+    data_lost = sum(~f_na) < df_sub.shape[0]
+    if not data_lost:
+        return df
+    # Lost data during join(). Manually check each df_sub.index date against df_main.index to
+    # find matching interval
+    df_sub = df_sub_backup.copy()
+    new_index = [-1]*df_sub.shape[0]
+    for i in range(df_sub.shape[0]):
+        dt_sub_i = df_sub.index[i]
+        if dt_sub_i in df_main.index:
+            new_index[i] = dt_sub_i ; continue
+        # Found a bad index date, need to search for near-match in df_main (same week/month)
+        fixed = False
+        for j in range(df_main.shape[0]-1):
+            dt_main_j0 = df_main.index[j]
+            dt_main_j1 = df_main.index[j+1]
+            if (dt_main_j0 <= dt_sub_i) and (dt_sub_i < dt_main_j1):
+                dt_sub_i = dt_main_j0 ; fixed = True ; break
+        if not fixed:
+            last_main_dt = df_main.index[df_main.shape[0]-1]
+            diff = dt_sub_i - last_main_dt
+            if interval == "1mo" and last_main_dt.month == dt_sub_i.month:
+                dt_sub_i = last_main_dt ; fixed = True
+            elif interval == "3mo" and last_main_dt.year == dt_sub_i.year and last_main_dt.quarter == dt_sub_i.quarter:
+                dt_sub_i = last_main_dt ; fixed = True
+            elif interval == "1wk" and last_main_dt.week == dt_sub_i.week:
+                dt_sub_i = last_main_dt ; fixed = True
+            elif interval == "1d" and last_main_dt.day == dt_sub_i.day:
+                dt_sub_i = last_main_dt ; fixed = True
+            elif interval == "1h" and last_main_dt.hour == dt_sub_i.hour:
+                dt_sub_i = last_main_dt ; fixed = True
+            else:
+                td = _pd.to_timedelta(interval)
+                if (dt_sub_i-last_main_dt) < td:
+                    dt_sub_i = last_main_dt ; fixed = True
+        if not fixed:
+            raise Exception("df_sub table contains row that failed to map to row in main table")
+        new_index[i] = dt_sub_i
+    df_sub = _reindex_events(df_sub, new_index, data_col)
+    df = df_main.join(df_sub)
+
+    f_na = df[data_col].isna()
+    data_lost = sum(~f_na) < df_sub.shape[0]
+    if data_lost:
+        raise Exception("Lost data during merge despite all attempts to align data")
+
+    return df
+
+
 def fix_Yahoo_dst_issue(df, interval):
     if interval in ["1d","1w","1wk"]:
         # These intervals should start at time 00:00. But for some combinations of date and timezone, 


Test patch:
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
--- /dev/null
+++ b/tests/__init__.py
@@ -0,0 +1 @@
+#!/usr/bin/env python
diff --git a/tests/context.py b/tests/context.py
new file mode 100644
--- /dev/null
+++ b/tests/context.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+_parent_dp = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+_src_dp = _parent_dp
+sys.path.insert(0, _src_dp)
+
+import yfinance
diff --git a/tests/prices.py b/tests/prices.py
new file mode 100644
--- /dev/null
+++ b/tests/prices.py
@@ -0,0 +1,44 @@
+from .context import yfinance as yf
+
+import unittest
+
+class TestPriceHistory(unittest.TestCase):
+	def setUp(self):
+		pass
+
+	def tearDown(self):
+		pass
+
+	def test_weeklyWithEvents(self):
+		# Reproduce issue #521
+		tkr1 = "QQQ"
+		tkr2 = "GDX"
+		start_d = "2014-12-29"
+		end_d = "2020-11-29"
+		df1 = yf.Ticker(tkr1).history(start=start_d, end=end_d, interval="1wk", actions=True)
+		df2 = yf.Ticker(tkr2).history(start=start_d, end=end_d, interval="1wk", actions=True)
+		try:
+			self.assertTrue(df1.index.equals(df2.index))
+		except:
+			missing_from_df1 = df2.index.difference(df1.index)
+			missing_from_df2 = df1.index.difference(df2.index)
+			print("{} missing these dates: {}".format(tkr1, missing_from_df1))
+			print("{} missing these dates: {}".format(tkr2, missing_from_df2))
+			raise
+
+		# Test that index same with and without events:
+		tkrs = [tkr1, tkr2]
+		for tkr in tkrs:
+			df1 = yf.Ticker(tkr).history(start=start_d, end=end_d, interval="1wk", actions=True)
+			df2 = yf.Ticker(tkr).history(start=start_d, end=end_d, interval="1wk", actions=False)
+			try:
+				self.assertTrue(df1.index.equals(df2.index))
+			except:
+				missing_from_df1 = df2.index.difference(df1.index)
+				missing_from_df2 = df1.index.difference(df2.index)
+				print("{}-with-events missing these dates: {}".format(tkr, missing_from_df1))
+				print("{}-without-events missing these dates: {}".format(tkr, missing_from_df2))
+				raise
+
+if __name__ == '__main__':
+	unittest.main()