Repository: streamlit/streamlit

Issue:
Implicit namespace packages are not detected by file watchers
Changes in [implicit namespace packages](https://www.python.org/dev/peps/pep-0420/) are currently not properly detected by `LocalSourcesWatcher.update_watched_modules`, as it can be evidenced by the following minimal example:

Download [minimal_example.zip](https://github.com/streamlit/streamlit/files/6178449/minimal_example.zip), and run the following on a Python 3.7 installation
```
cd minimal_example
streamlit run example.py
```
This should print a simple "Hello" on your browser.
If you change `minimal_example/my_namespace_package/my_file.py` to `MESSAGE = "Hello again!"` and click rerun on streamlit's webpage, the hello message will not get updated.



Discussion:

Resolving patch:
diff --git a/lib/streamlit/watcher/local_sources_watcher.py b/lib/streamlit/watcher/local_sources_watcher.py
--- a/lib/streamlit/watcher/local_sources_watcher.py
+++ b/lib/streamlit/watcher/local_sources_watcher.py
@@ -15,9 +15,12 @@
 import os
 import sys
 import collections
+import typing as t
+import types
 
 from streamlit import config
 from streamlit import file_util
+from streamlit import warning
 from streamlit.folder_black_list import FolderBlackList
 
 from streamlit.logger import get_logger
@@ -25,7 +28,6 @@
 
 LOGGER = get_logger(__name__)
 
-
 FileWatcher = get_default_file_watcher_class()
 
 WatchedModule = collections.namedtuple("WatchedModule", ["watcher", "module_name"])
@@ -120,60 +122,62 @@ def update_watched_modules(self):
         if self._is_closed:
             return
 
-        local_filepaths = []
-
-        # Clone modules dict here because we may alter the original dict inside
-        # the loop.
-        modules = dict(sys.modules)
-
-        for name, module in modules.items():
-            try:
-                spec = getattr(module, "__spec__", None)
-
-                if spec is None:
-                    filepath = getattr(module, "__file__", None)
-                    if filepath is None:
-                        # Some modules have neither a spec nor a file. But we
-                        # can ignore those since they're not the user-created
-                        # modules we want to watch anyway.
-                        continue
-                else:
-                    filepath = spec.origin
-
-                if filepath is None:
-                    # Built-in modules (and other stuff) don't have origins.
-                    continue
-
-                filepath = os.path.abspath(filepath)
-
-                if not os.path.isfile(filepath):
-                    # There are some modules that have a .origin, but don't
-                    # point to real files. For example, there's a module where
-                    # .origin is 'built-in'.
-                    continue
-
-                if self._folder_black_list.is_blacklisted(filepath):
-                    continue
-
-                local_filepaths.append(filepath)
-
-                if self._file_should_be_watched(filepath):
-                    self._register_watcher(filepath, name)
-
-            except Exception:
-                # In case there's a problem introspecting some specific module,
-                # let's not stop the entire loop from running.  For example,
-                # the __spec__ field in some modules (like IPython) is actually
-                # a dynamic property, which can crash if the underlying
-                # module's code has a bug (as discovered by one of our users).
-                continue
-
-        # Clone dict here because we may alter the original dict inside the
-        # loop.
-        watched_modules = dict(self._watched_modules)
-
-        # Remove no-longer-depended-on files from self._watched_modules
-        # Will this ever happen?
-        for filepath in watched_modules:
-            if filepath not in local_filepaths:
-                self._deregister_watcher(filepath)
+        modules_paths = {
+            name: self._exclude_blacklisted_paths(get_module_paths(module))
+            for name, module in dict(sys.modules).items()
+        }
+
+        self._register_necessary_watchers(modules_paths)
+
+    def _register_necessary_watchers(
+        self, module_paths: t.Dict[str, t.Set[str]]
+    ) -> None:
+        for name, paths in module_paths.items():
+            for path in paths:
+                if self._file_should_be_watched(path):
+                    self._register_watcher(path, name)
+
+    def _exclude_blacklisted_paths(self, paths: t.Set[str]) -> t.Set[str]:
+        return {p for p in paths if not self._folder_black_list.is_blacklisted(p)}
+
+
+def get_module_paths(module: types.ModuleType) -> t.Set[str]:
+    paths_extractors = [
+        # https://docs.python.org/3/reference/datamodel.html
+        # __file__ is the pathname of the file from which the module was loaded
+        # if it was loaded from a file.
+        # The __file__ attribute may be missing for certain types of modules
+        lambda m: [m.__file__],
+        # https://docs.python.org/3/reference/import.html#__spec__
+        # The __spec__ attribute is set to the module spec that was used
+        # when importing the module. one exception is __main__,
+        # where __spec__ is set to None in some cases.
+        # https://www.python.org/dev/peps/pep-0451/#id16
+        # "origin" in an import context means the system
+        # (or resource within a system) from which a module originates
+        # ... It is up to the loader to decide on how to interpret
+        # and use a module's origin, if at all.
+        lambda m: [m.__spec__.origin],
+        # https://www.python.org/dev/peps/pep-0420/
+        # Handling of "namespace packages" in which the __path__ attribute
+        # is a _NamespacePath object with a _path attribute containing
+        # the various paths of the package.
+        lambda m: [p for p in m.__path__._path]
+    ]
+
+    all_paths = set()
+    for extract_paths in paths_extractors:
+        potential_paths = []
+        try:
+            potential_paths = extract_paths(module)
+        except AttributeError:
+            pass
+        except Exception as e:
+            warning(f"Examining the path of {module.__name__} raised {e}")
+
+        all_paths.update([str(p) for p in potential_paths if _is_valid_path(p)])
+    return all_paths
+
+
+def _is_valid_path(path: t.Optional[str]) -> bool:
+    return isinstance(path, str) and (os.path.isfile(path) or os.path.isdir(path))
diff --git a/lib/streamlit/watcher/util.py b/lib/streamlit/watcher/util.py
--- a/lib/streamlit/watcher/util.py
+++ b/lib/streamlit/watcher/util.py
@@ -20,6 +20,7 @@
 
 import hashlib
 import time
+import os
 
 
 # How many times to try to grab the MD5 hash.
@@ -30,7 +31,7 @@
 
 
 def calc_md5_with_blocking_retries(file_path):
-    """Calculate the MD5 checksum of the given file.
+    """Calculate the MD5 checksum of a given file_path
 
     IMPORTANT: This method calls time.sleep(), which blocks execution. So you
     should only use this outside the main thread.
@@ -46,23 +47,31 @@ def calc_md5_with_blocking_retries(file_path):
         The MD5 checksum.
 
     """
-    file_bytes = b""
 
+    if os.path.isdir(file_path):
+        content = file_path.encode("UTF8")
+    else:
+        content = _get_file_content_with_blocking_retries(file_path)
+
+    md5 = hashlib.md5()
+    md5.update(content)
+
+    # Use hexdigest() instead of digest(), so it's easier to debug.
+    return md5.hexdigest()
+
+
+def _get_file_content_with_blocking_retries(file_path):
+    content = b""
     # There's a race condition where sometimes file_path no longer exists when
     # we try to read it (since the file is in the process of being written).
     # So here we retry a few times using this loop. See issue #186.
     for i in range(_MAX_RETRIES):
         try:
             with open(file_path, "rb") as f:
-                file_bytes = f.read()
+                content = f.read()
                 break
         except FileNotFoundError as e:
             if i >= _MAX_RETRIES - 1:
                 raise e
             time.sleep(_RETRY_WAIT_SECS)
-
-    md5 = hashlib.md5()
-    md5.update(file_bytes)
-
-    # Use hexdigest() instead of digest(), so it's easier to debug.
-    return md5.hexdigest()
+    return content


Test patch:
diff --git a/lib/tests/streamlit/watcher/local_sources_watcher_test.py b/lib/tests/streamlit/watcher/local_sources_watcher_test.py
--- a/lib/tests/streamlit/watcher/local_sources_watcher_test.py
+++ b/lib/tests/streamlit/watcher/local_sources_watcher_test.py
@@ -247,6 +247,27 @@ def test_config_watcherType(self, _):
                 "PollingFileWatcher",
             )
 
+    @patch("streamlit.watcher.local_sources_watcher.FileWatcher")
+    def test_namespace_package_unloaded(self, fob, _):
+        import tests.streamlit.watcher.test_data.namespace_package as pkg
+
+        pkg_path = os.path.abspath(pkg.__path__._path[0])
+
+        lsw = local_sources_watcher.LocalSourcesWatcher(REPORT, NOOP_CALLBACK)
+
+        fob.assert_called_once()
+
+        with patch("sys.modules", {"pkg": pkg}):
+            lsw.update_watched_modules()
+
+            # Simulate a change to the child module
+            lsw.on_file_changed(pkg_path)
+
+            # Assert that both the parent and child are unloaded, ready for reload
+            self.assertNotIn("pkg", sys.modules)
+
+        del sys.modules["tests.streamlit.watcher.test_data.namespace_package"]
+
 
 def sort_args_list(args_list):
     return sorted(args_list, key=lambda args: args[0])
diff --git a/lib/tests/streamlit/watcher/test_data/namespace_package/empty_file.py b/lib/tests/streamlit/watcher/test_data/namespace_package/empty_file.py
new file mode 100644