Repository: databrickslabs/ucx

Issue:
[BUG]: Some unit test fail when running without internet access
### Is there an existing issue for this?

- [X] I have searched the existing issues

### Current Behavior

Some unit test fail when running without internet access

### Expected Behavior

Unit tests should run independent of internet access

### Steps To Reproduce

1. Turn off your internet access
2. Run unit test suite

### Cloud

Azure

### Operating System

macOS

### Version

latest via Databricks CLI

### Relevant log output

_No response_

Discussion:

Solution patch:
diff --git a/src/databricks/labs/ucx/assessment/dashboards.py b/src/databricks/labs/ucx/assessment/dashboards.py
index 67fe12e6..66a472c8 100644
--- a/src/databricks/labs/ucx/assessment/dashboards.py
+++ b/src/databricks/labs/ucx/assessment/dashboards.py
@@ -310,11 +310,13 @@ class LakeviewDashboardCrawler(CrawlerBase[Dashboard]):
         schema: str,
         *,
         include_dashboard_ids: list[str] | None = None,
+        exclude_dashboard_ids: list[str] | None = None,
         include_query_ids: list[str] | None = None,
     ):
         super().__init__(sql_backend, "hive_metastore", schema, "lakeview_dashboards", Dashboard)
         self._ws = ws
         self._include_dashboard_ids = include_dashboard_ids
+        self._exclude_dashboard_ids = exclude_dashboard_ids
         self._include_query_ids = include_query_ids
 
     def _crawl(self) -> Iterable[Dashboard]:
@@ -322,6 +324,8 @@ class LakeviewDashboardCrawler(CrawlerBase[Dashboard]):
         for sdk_dashboard in self._list_dashboards():
             if sdk_dashboard.dashboard_id is None:
                 continue
+            if sdk_dashboard.dashboard_id in (self._exclude_dashboard_ids or []):
+                continue
             dashboard = Dashboard.from_sdk_lakeview_dashboard(sdk_dashboard)
             dashboards.append(dashboard)
         return dashboards
diff --git a/src/databricks/labs/ucx/assessment/sequencing.py b/src/databricks/labs/ucx/assessment/sequencing.py
index 4fa660f5..4af02f00 100644
--- a/src/databricks/labs/ucx/assessment/sequencing.py
+++ b/src/databricks/labs/ucx/assessment/sequencing.py
@@ -107,7 +107,7 @@ class PriorityQueue:
 
     _REMOVED = "<removed>"  # Mark removed items
 
-    def __init__(self):
+    def __init__(self) -> None:
         self._entries: list[QueueEntry] = []
         self._entry_finder: dict[MigrationNode, QueueEntry] = {}
         self._counter = itertools.count()  # Tiebreaker with equal priorities, then "first in, first out"
@@ -120,7 +120,7 @@ class PriorityQueue:
         if task in self._entry_finder:
             self._remove(task)
         count = next(self._counter)
-        entry = [priority, count, task]
+        entry: QueueEntry = [priority, count, task]
         self._entry_finder[task] = entry
         heapq.heappush(self._entries, entry)
 
@@ -138,6 +138,7 @@ class PriorityQueue:
     def _remove(self, task: MigrationNode) -> None:
         """Remove a task from the queue."""
         entry = self._entry_finder.pop(task)
+        # The entry is also stored in self._entries.
         entry[2] = self._REMOVED
 
 
diff --git a/src/databricks/labs/ucx/contexts/application.py b/src/databricks/labs/ucx/contexts/application.py
index 65a0c7a8..102fcdb4 100644
--- a/src/databricks/labs/ucx/contexts/application.py
+++ b/src/databricks/labs/ucx/contexts/application.py
@@ -303,6 +303,7 @@ class GlobalContext(abc.ABC):
             self.sql_backend,
             self.inventory_database,
             include_dashboard_ids=self.config.include_dashboard_ids,
+            exclude_dashboard_ids=list(self.install_state.dashboards.values()),
             include_query_ids=self.config.include_query_ids,
         )
 


Test patch:
diff --git a/tests/unit/assessment/test_dashboards.py b/tests/unit/assessment/test_dashboards.py
index cf0ae8f7..b9ffa948 100644
--- a/tests/unit/assessment/test_dashboards.py
+++ b/tests/unit/assessment/test_dashboards.py
@@ -458,6 +458,36 @@ def test_lakeview_dashboard_crawler_skips_not_found_dashboard_ids(caplog, mock_b
     ws.lakeview.list.assert_not_called()
 
 
+def test_lakeview_dashboard_crawler_skips_exclude_dashboard_ids(caplog, mock_backend) -> None:
+    ws = create_autospec(WorkspaceClient)
+    dashboards = [SdkLakeviewDashboard(dashboard_id="did1"), SdkLakeviewDashboard(dashboard_id="did2")]
+    ws.lakeview.list.side_effect = lambda: (dashboard for dashboard in dashboards)  # Expects an iterator
+    crawler = LakeviewDashboardCrawler(ws, mock_backend, "test", exclude_dashboard_ids=["did2"])
+
+    crawler.snapshot()
+
+    rows = mock_backend.rows_written_for("hive_metastore.test.lakeview_dashboards", "overwrite")
+    assert rows == [Row(id="did1", name=None, parent=None, query_ids=[], tags=[], creator_id=None)]
+    ws.lakeview.list.assert_called_once()
+
+
+def test_lakeview_dashboard_crawler_skips_exclude_dashboard_ids_takes_priority_over_include_dashboard_ids(
+    caplog, mock_backend
+) -> None:
+    ws = create_autospec(WorkspaceClient)
+    ws.lakeview.get.side_effect = lambda dashboard_id: SdkLakeviewDashboard(dashboard_id=dashboard_id)
+    crawler = LakeviewDashboardCrawler(
+        ws, mock_backend, "test", include_dashboard_ids=["did1", "did2"], exclude_dashboard_ids=["did2"]
+    )
+
+    crawler.snapshot()
+
+    rows = mock_backend.rows_written_for("hive_metastore.test.lakeview_dashboards", "overwrite")
+    assert rows == [Row(id="did1", name=None, parent=None, query_ids=[], tags=[], creator_id=None)]
+    ws.lakeview.list.assert_not_called()
+    ws.lakeview.get.assert_has_calls([call("did1"), call("did2")])
+
+
 def test_lakeview_dashboard_crawler_list_queries_includes_query_ids(mock_backend) -> None:
     ws = create_autospec(WorkspaceClient)
     datasets = [
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
index db69a7d7..0500f87c 100644
--- a/tests/unit/conftest.py
+++ b/tests/unit/conftest.py
@@ -223,8 +223,17 @@ def ws():
 
 @pytest.fixture
 def simple_dependency_resolver(mock_path_lookup: PathLookup) -> DependencyResolver:
+
+    def mock_pip_install_always_successful(_) -> tuple[int, str, str]:
+        """Mock an always successful pip install.
+
+        Pip installs require internet access which we want to avoid during unit testing for speed and reliability.
+        While using the simple dependency resolver, we mock any library install as successful.
+        """
+        return 0, "", ""
+
     allow_list = KnownList()
-    library_resolver = PythonLibraryResolver(allow_list)
+    library_resolver = PythonLibraryResolver(allow_list, mock_pip_install_always_successful)
     notebook_resolver = NotebookResolver(NotebookLoader())
     import_resolver = ImportFileResolver(FileLoader(), allow_list)
     return DependencyResolver(library_resolver, notebook_resolver, import_resolver, import_resolver, mock_path_lookup)
diff --git a/tests/unit/source_code/test_graph.py b/tests/unit/source_code/test_graph.py
index 84cd5850..8644d525 100644
--- a/tests/unit/source_code/test_graph.py
+++ b/tests/unit/source_code/test_graph.py
@@ -4,40 +4,28 @@ from pathlib import Path
 import pytest
 
 from databricks.labs.ucx.source_code.base import CurrentSessionState
-from databricks.labs.ucx.source_code.linters.files import FileLoader, ImportFileResolver, FolderLoader
+from databricks.labs.ucx.source_code.linters.files import FileLoader, FolderLoader
 from databricks.labs.ucx.source_code.graph import (
     Dependency,
     DependencyGraph,
-    DependencyResolver,
     InheritedContext,
     DependencyGraphWalker,
 )
-from databricks.labs.ucx.source_code.notebooks.loaders import NotebookResolver, NotebookLoader
+from databricks.labs.ucx.source_code.notebooks.loaders import NotebookLoader
 from databricks.labs.ucx.source_code.path_lookup import PathLookup
 from databricks.labs.ucx.source_code.python.python_ast import Tree
-from databricks.labs.ucx.source_code.python_libraries import PythonLibraryResolver
-from databricks.labs.ucx.source_code.known import KnownList
 
 
-def test_dependency_graph_registers_library(mock_path_lookup) -> None:
+def test_dependency_graph_registers_library_from_egg(mock_path_lookup, simple_dependency_resolver) -> None:
+    egg_path = Path(__file__).parent / "samples/distribution/dist/thingy-0.0.1-py3.10.egg"
     dependency = Dependency(FileLoader(), Path("test"))
-    file_loader = FileLoader()
-    allow_list = KnownList()
     session_state = CurrentSessionState()
-    import_file_resolver = ImportFileResolver(file_loader, allow_list)
-    dependency_resolver = DependencyResolver(
-        PythonLibraryResolver(allow_list),
-        NotebookResolver(NotebookLoader()),
-        import_file_resolver,
-        import_file_resolver,
-        mock_path_lookup,
-    )
-    graph = DependencyGraph(dependency, None, dependency_resolver, mock_path_lookup, session_state)
+    graph = DependencyGraph(dependency, None, simple_dependency_resolver, mock_path_lookup, session_state)
 
-    problems = graph.register_library("demo-egg")  # installs pkgdir
+    problems = graph.register_library(egg_path.as_posix())
 
     assert len(problems) == 0
-    lookup_resolve = graph.path_lookup.resolve(Path("pkgdir"))
+    lookup_resolve = graph.path_lookup.resolve(Path("thingy"))
     assert lookup_resolve is not None
     assert lookup_resolve.exists()
 
diff --git a/tests/unit/source_code/test_jobs.py b/tests/unit/source_code/test_jobs.py
index d067b3b0..d5b53184 100644
--- a/tests/unit/source_code/test_jobs.py
+++ b/tests/unit/source_code/test_jobs.py
@@ -88,19 +88,6 @@ def test_workflow_task_container_builds_dependency_graph_empty_task(mock_path_lo
     ws.assert_not_called()
 
 
-def test_workflow_task_container_builds_dependency_graph_pytest_pypi_library(mock_path_lookup, graph) -> None:
-    ws = create_autospec(WorkspaceClient)
-    libraries = [compute.Library(pypi=compute.PythonPyPiLibrary(package="demo-egg"))]  # installs pkgdir
-    task = jobs.Task(task_key="test", libraries=libraries)
-
-    workflow_task_container = WorkflowTaskContainer(ws, task, Job())
-    problems = workflow_task_container.build_dependency_graph(graph)
-
-    assert len(problems) == 0
-    assert graph.path_lookup.resolve(Path("pkgdir")).exists()
-    ws.assert_not_called()
-
-
 def test_workflow_task_container_builds_dependency_graph_unknown_pypi_library(mock_path_lookup, graph) -> None:
     ws = create_autospec(WorkspaceClient)
     libraries = [compute.Library(pypi=compute.PythonPyPiLibrary(package="unknown-library-name"))]
diff --git a/tests/unit/source_code/test_known.py b/tests/unit/source_code/test_known.py
index 2d5444f4..641b0ecc 100644
--- a/tests/unit/source_code/test_known.py
+++ b/tests/unit/source_code/test_known.py
@@ -73,6 +73,7 @@ def test_error_on_missing_known_json() -> None:
         KnownList._get_known()  # pylint: disable=protected-access
 
 
+@pytest.mark.skip(reason="Interferes with tests reading known list at the same time of rebuilding (writing)")
 def test_rebuild_trivial() -> None:
     # No-op: the known.json file is already up-to-date
     cwd = Path.cwd()