Repository: ranaroussi/yfinance

Issue:
0.2.29 bug: OperationalError('unable to open database file')
### Describe bug

fail when downloading array, work ok in 0.2.28

### Simple code that reproduces your problem

``` Python
from pandas_datareader import data as pdr
import yfinance as yf
from numpy import isnan

##yf.pdr_override()

def loadtickers(tickers,start="2005-1-1",end=None,renamefield={},prev=None):
    for i in range(0,2):
        try:
            df = yf.download(tickers, start, end, auto_adjust=True,ignore_tz = True)
            df.columns = ['_'.join(((str.lower(col[1]) if col[1] not in renamefield.keys() else str.lower(renamefield[col[1]]), str.lower(col[0])))) for col in df.columns]
            for t in tickers:
                if t in renamefield.keys():
                    t=renamefield[t]
                print("{}_return".format(t).lower(),"{}_close".format(t).lower() )
                df["{}_return".format(t).lower()] = df["{}_close".format(t).lower()].pct_change().fillna(0)
            if prev is not None:
                df=df.join(prev,how='left')
            return df
        except Exception as e:
            print(e)
            continue
        break
    return

p1=loadtickers(["SPY","TLT"],start="2004-12-31",renamefield={"^VXTLT":"VXTLT","^TNX":"TNX",})
```

### Debug log
DEBUG    Entering download()
DEBUG:yfinance:Entering download()
DEBUG     Disabling multithreading because DEBUG logging enabled
DEBUG:yfinance: Disabling multithreading because DEBUG logging enabled
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
DEBUG     Entering history()
DEBUG:yfinance: Entering history()
ERROR     
          24 Failed downloads:
ERROR:yfinance: 
 24 Failed downloads:
ERROR     ['TMF', 'TMV', 'VBINX', 'SHV', '^GVZ', 'VXZ', '^VIX', 'SGOV', 'TLT', 'QQQ', '^VXN', 'GLL', 'SPHD', 'SPY', 'TQQQ', 'QLD', 'SPLV', 'TBF', 'IAU', '^VXTLT', 'GLD', 'XLU', 'VXX', 'UGL']: OperationalError('unable to open database file')
ERROR:yfinance: ['TMF', 'TMV', 'VBINX', 'SHV', '^GVZ', 'VXZ', '^VIX', 'SGOV', 'TLT', 'QQQ', '^VXN', 'GLL', 'SPHD', 'SPY', 'TQQQ', 'QLD', 'SPLV', 'TBF', 'IAU', '^VXTLT', 'GLD', 'XLU', 'VXX', 'UGL']: OperationalError('unable to open database file')
DEBUG     ['TMF', 'TMV', 'VBINX', 'SHV', '^GVZ', 'VXZ', '^VIX', 'SGOV', 'TLT', 'QQQ', '^VXN', 'GLL', 'SPHD', 'SPY', 'TQQQ', 'QLD', 'SPLV', 'TBF', 'IAU', '^VXTLT', 'GLD', 'XLU', 'VXX', 'UGL']: Traceback (most recent call last):
            File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 3192, in connect
              self._state.set_connection(self._connect())
            File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 3536, in _connect
              conn = sqlite3.connect(self.database, timeout=self._timeout,
          sqlite3.OperationalError: unable to open database file
          
          During handling of the above exception, another exception occurred:
          
          Traceback (most recent call last):
            File "/usr/local/lib/python3.10/dist-packages/yfinance/multi.py", line 275, in _download_one
              data = Ticker(ticker, session=session).history(
            File "/usr/local/lib/python3.10/dist-packages/yfinance/utils.py", line 114, in wrapper
              result = func(*args, **kwargs)
            File "/usr/local/lib/python3.10/dist-packages/yfinance/base.py", line 148, in history
              tz = self._get_ticker_tz(proxy, timeout)
            File "/usr/local/lib/python3.10/dist-packages/yfinance/base.py", line 1644, in _get_ticker_tz
              cache = utils.get_tz_cache()
            File "/usr/local/lib/python3.10/dist-packages/yfinance/utils.py", line 985, in get_tz_cache
              _tz_cache = _TzCache()
            File "/usr/local/lib/python3.10/dist-packages/yfinance/utils.py", line 926, in __init__
              db.connect()
            File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 3191, in connect
              with __exception_wrapper__:
            File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 3019, in __exit__
              reraise(new_type, new_type(exc_value, *exc_args), traceback)
            File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 192, in reraise
              raise value.with_traceback(tb)
            File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 3192, in connect
              self._state.set_connection(self._connect())
            File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 3536, in _connect
              conn = sqlite3.connect(self.database, timeout=self._timeout,
          peewee.OperationalError: unable to open database file
          
DEBUG:yfinance: ['TMF', 'TMV', 'VBINX', 'SHV', '^GVZ', 'VXZ', '^VIX', 'SGOV', 'TLT', 'QQQ', '^VXN', 'GLL', 'SPHD', 'SPY', 'TQQQ', 'QLD', 'SPLV', 'TBF', 'IAU', '^VXTLT', 'GLD', 'XLU', 'VXX', 'UGL']: Traceback (most recent call last):
   File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 3192, in connect
     self._state.set_connection(self._connect())
   File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 3536, in _connect
     conn = sqlite3.connect(self.database, timeout=self._timeout,
 sqlite3.OperationalError: unable to open database file
 
 During handling of the above exception, another exception occurred:
 
 Traceback (most recent call last):
   File "/usr/local/lib/python3.10/dist-packages/yfinance/multi.py", line 275, in _download_one
     data = Ticker(ticker, session=session).history(
   File "/usr/local/lib/python3.10/dist-packages/yfinance/utils.py", line 114, in wrapper
     result = func(*args, **kwargs)
   File "/usr/local/lib/python3.10/dist-packages/yfinance/base.py", line 148, in history
     tz = self._get_ticker_tz(proxy, timeout)
   File "/usr/local/lib/python3.10/dist-packages/yfinance/base.py", line 1644, in _get_ticker_tz
     cache = utils.get_tz_cache()
   File "/usr/local/lib/python3.10/dist-packages/yfinance/utils.py", line 985, in get_tz_cache
     _tz_cache = _TzCache()
   File "/usr/local/lib/python3.10/dist-packages/yfinance/utils.py", line 926, in __init__
     db.connect()
   File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 3191, in connect
     with __exception_wrapper__:
   File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 3019, in __exit__
     reraise(new_type, new_type(exc_value, *exc_args), traceback)
   File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 192, in reraise
     raise value.with_traceback(tb)
   File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 3192, in connect
     self._state.set_connection(self._connect())
   File "/usr/local/lib/python3.10/dist-packages/peewee.py", line 3536, in _connect
     conn = sqlite3.connect(self.database, timeout=self._timeout,
 peewee.OperationalError: unable to open database file
 
DEBUG    Exiting download()
DEBUG:yfinance:Exiting download()
2023-09-22-15-59 tickers.pkl
loading fresh 2023-09-22-15-59 tickers.pkl
spy_return spy_close
tlt_return tlt_close
iau_return iau_close
gld_return gld_close
qqq_return qqq_close
vix_return vix_close
vxtlt_return vxtlt_close
gvz_return gvz_close
qld_return qld_close
vxz_return vxz_close
tbf_return tbf_close
vxn_return vxn_close
vxx_return vxx_close
tmf_return tmf_close
tmv_return tmv_close
splv_return splv_close
sphd_return sphd_close
vbinx_return vbinx_close
xlu_return xlu_close
tqqq_return tqqq_close
ugl_return ugl_close
gll_return gll_close
sgov_return sgov_close
shv_return shv_close

### Bad data proof

_No response_

### `yfinance` version

0.2.29

### Python version

_No response_

### Operating system

_No response_


Discussion:
You plan on editing with error message and working code?
i added, tell me of this is enough, it's fail with 0.2.29, but ok with 0.2.28
Run again debug logging enabled. Like the bug report template instructed.
done!
same
I can't debug until tomorrow, but here's summary in case someone else can.

Previous version was 99% thread-safe. In rare cases the thread-safety failed.

Latest version switches to `peewee` module. Supposed to be 100% thread-safe, and I tested on few threads. Maybe it's configured wrong?
Confirmed. Fails on my CI pipeline in python `<=3.10`, works on my mac with `3.11`.
There's clearly a bug in 0.2.29. Keep replies constructive and concise.
@enzbus how many CPUs on your mac, vs how many on system running pipeline?
My sample fail on Google colab 
> @enzbus how many CPUs on your mac, vs how many on system running pipeline?

So far I've only observed failure on github CI, which is single processor (I believe). My machine has multiple processors. The only way I call `yfinance` is via `yf.download`, from a single thread. The errors that I get on github testing are like this

> ERROR:yfinance:
> 1 Failed download:
> ERROR:yfinance:['AAPL']: OperationalError('unable to open database file')

(And then `yf.download` returns an empty dataframe, I believe.)
I am encountering the same problem, both on python 3.9 and 3.11. FWIW, the machine I'm running uses Ryzen 5 5600X CPU.
Not sure if this is related, but I am having a similar issue when using `yf.download`, I am getting a `sqlite3.OperationalError: unable to open database file` error. I am running on Windows and I believe it may have something to do with the cache path folder not existing.

If I manually create the `py-yfinance` folder in `C:\Users\Himel\AppData\Local\`, I no longer get the exception.

Resolving patch:
diff --git a/yfinance/utils.py b/yfinance/utils.py
--- a/yfinance/utils.py
+++ b/yfinance/utils.py
@@ -902,74 +902,132 @@ def __str__(self):
 # ---------------------------------
 
 
-_cache_dir = _os.path.join(_ad.user_cache_dir(), "py-yfinance")
-DB_PATH = _os.path.join(_cache_dir, 'tkr-tz.db')
-db = _peewee.SqliteDatabase(DB_PATH, pragmas={'journal_mode': 'wal', 'cache_size': -64})
-_tz_cache = None
+_cache_init_lock = Lock()
 
 
 class _TzCacheException(Exception):
     pass
 
 
-class KV(_peewee.Model):
+class _TzCacheDummy:
+    """Dummy cache to use if tz cache is disabled"""
+
+    def lookup(self, tkr):
+        return None
+
+    def store(self, tkr, tz):
+        pass
+
+    @property
+    def tz_db(self):
+        return None
+
+
+class _TzCacheManager:
+    _tz_cache = None
+
+    @classmethod
+    def get_tz(cls):
+        if cls._tz_cache is None:
+            cls._initialise()
+        return cls._tz_cache
+
+    @classmethod
+    def _initialise(cls, cache_dir=None):
+        try:
+            cls._tz_cache = _TzCache()
+        except _TzCacheException as err:
+            get_yf_logger().info(f"Failed to create TzCache, reason: {err}. "
+                                 "TzCache will not be used. "
+                                 "Tip: You can direct cache to use a different location with 'set_tz_cache_location(mylocation)'")
+            cls._tz_cache = _TzCacheDummy()
+
+
+class _DBManager:
+    _db = None
+    _cache_dir = _os.path.join(_ad.user_cache_dir(), "py-yfinance")
+
+    @classmethod
+    def get_database(cls):
+        if cls._db is None:
+            cls._initialise()
+        return cls._db
+
+    @classmethod
+    def close_db(cls):
+        if cls._db is not None:
+            try:
+                cls._db.close()
+            except Exception as e:
+                # Must discard exceptions because Python trying to quit.
+                pass
+
+
+    @classmethod
+    def _initialise(cls, cache_dir=None):
+        if cache_dir is not None:
+            cls._cache_dir = cache_dir
+
+        if not _os.path.isdir(cls._cache_dir):
+            _os.mkdir(cls._cache_dir)
+        cls._db = _peewee.SqliteDatabase(
+            _os.path.join(cls._cache_dir, 'tkr-tz.db'),
+            pragmas={'journal_mode': 'wal', 'cache_size': -64}
+        )
+
+        old_cache_file_path = _os.path.join(cls._cache_dir, "tkr-tz.csv")
+        if _os.path.isfile(old_cache_file_path):
+            _os.remove(old_cache_file_path)
+
+    @classmethod
+    def change_location(cls, new_cache_dir):
+        cls._db.close()
+        cls._db = None
+        cls._cache_dir = new_cache_dir
+# close DB when Python exists
+_atexit.register(_DBManager.close_db)
+
+
+class _KV(_peewee.Model):
     key = _peewee.CharField(primary_key=True)
     value = _peewee.CharField(null=True)
     
     class Meta:
-        database = db
+        database = _DBManager.get_database()
         without_rowid = True
 
 
 class _TzCache:
     def __init__(self):
+        db = _DBManager.get_database()
         db.connect()
-        db.create_tables([KV])
+        db.create_tables([_KV])
 
-        old_cache_file_path = _os.path.join(_cache_dir, "tkr-tz.csv")
-        if _os.path.isfile(old_cache_file_path):
-            _os.remove(old_cache_file_path)
 
     def lookup(self, key):
         try:
-            return KV.get(KV.key == key).value
-        except KV.DoesNotExist:
+            return _KV.get(_KV.key == key).value
+        except _KV.DoesNotExist:
             return None
 
     def store(self, key, value):
+        db = _DBManager.get_database()
         try:
             if value is None:
-                q = KV.delete().where(KV.key == key)
+                q = _KV.delete().where(_KV.key == key)
                 q.execute()
                 return
             with db.atomic():
-                KV.insert(key=key, value=value).execute()
-        except IntegrityError:
+                _KV.insert(key=key, value=value).execute()
+        except _peewee.IntegrityError:
             # Integrity error means the key already exists. Try updating the key.
             old_value = self.lookup(key)
             if old_value != value:
                 get_yf_logger().debug(f"Value for key {key} changed from {old_value} to {value}.")
                 with db.atomic():
-                    q = KV.update(value=value).where(KV.key == key)
+                    q = _KV.update(value=value).where(_KV.key == key)
                     q.execute()
 
-    def close(self):
-        db.close()
-
-
-class _TzCacheDummy:
-    """Dummy cache to use if tz cache is disabled"""
-
-    def lookup(self, tkr):
-        return None
-
-    def store(self, tkr, tz):
-        pass
-
-    @property
-    def tz_db(self):
-        return None
-
 
 def get_tz_cache():
     """
@@ -979,20 +1037,7 @@ def get_tz_cache():
     """
     # as this can be called from multiple threads, protect it.
     with _cache_init_lock:
-        global _tz_cache
-        if _tz_cache is None:
-            try:
-                _tz_cache = _TzCache()
-            except _TzCacheException as err:
-                get_yf_logger().info(f"Failed to create TzCache, reason: {err}. "
-                                     "TzCache will not be used. "
-                                     "Tip: You can direct cache to use a different location with 'set_tz_cache_location(mylocation)'")
-                _tz_cache = _TzCacheDummy()
-
-        return _tz_cache
-
-
-_cache_init_lock = Lock()
+        return _TzCacheManager.get_tz()
 
 
 def set_tz_cache_location(cache_dir: str):
@@ -1003,6 +1048,4 @@ def set_tz_cache_location(cache_dir: str):
     :param cache_dir: Path to use for caches
     :return: None
     """
-    global _cache_dir, _tz_cache
-    assert _tz_cache is None, "Time Zone cache already initialized, setting path must be done before cache is created"
-    _cache_dir = cache_dir
+    _DBManager.change_location(cache_dir)


Test patch:
diff --git a/tests/context.py b/tests/context.py
--- a/tests/context.py
+++ b/tests/context.py
@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 
+import appdirs as _ad
+import datetime as _dt
 import sys
 import os
 _parent_dp = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
@@ -14,10 +16,17 @@
 # logging.basicConfig(level=logging.DEBUG)
 
 
+# Use adjacent cache folder for testing, delete if already exists and older than today
+testing_cache_dirpath = os.path.join(_ad.user_cache_dir(), "py-yfinance-testing")
+yfinance.set_tz_cache_location(testing_cache_dirpath)
+if os.path.isdir(testing_cache_dirpath):
+    mtime = _dt.datetime.fromtimestamp(os.path.getmtime(testing_cache_dirpath))
+    if mtime.date() < _dt.date.today():
+        import shutil
+        shutil.rmtree(testing_cache_dirpath)
+
+
 # Setup a session to rate-limit and cache persistently:
-import datetime as _dt
-import os
-import appdirs as _ad
 from requests import Session
 from requests_cache import CacheMixin, SQLiteCache
 from requests_ratelimiter import LimiterMixin, MemoryQueueBucket
@@ -26,12 +35,7 @@ class CachedLimiterSession(CacheMixin, LimiterMixin, Session):
 from pyrate_limiter import Duration, RequestRate, Limiter
 history_rate = RequestRate(1, Duration.SECOND*2)
 limiter = Limiter(history_rate)
-cache_fp = os.path.join(_ad.user_cache_dir(), "py-yfinance", "unittests-cache")
-if os.path.isfile(cache_fp + '.sqlite'):
-    # Delete local cache if older than 1 day:
-    mod_dt = _dt.datetime.fromtimestamp(os.path.getmtime(cache_fp + '.sqlite'))
-    if mod_dt.date() < _dt.date.today():
-        os.remove(cache_fp + '.sqlite')
+cache_fp = os.path.join(testing_cache_dirpath, "unittests-cache")
 session_gbl = CachedLimiterSession(
     limiter=limiter,
     bucket_class=MemoryQueueBucket,