Repository: altair-viz/altair
Pull request #3102 (2023-07-10 20:15:05)

Add support to transformed_data for reconstructed charts (with from_dict/from_json)
Addresses https://github.com/hex-inc/vegafusion/issues/354 on the Altair side. Port to VegaFusion will be done by @jonmmease.


Patch:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 37995c13..bc4774cb 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
         jsonschema-version: ["3.0", "4.17"]
     name: py ${{ matrix.python-version }} js ${{ matrix.jsonschema-version }}
     steps:
diff --git a/altair/utils/_transformed_data.py b/altair/utils/_transformed_data.py
index cf042c3f..f2f0af04 100644
--- a/altair/utils/_transformed_data.py
+++ b/altair/utils/_transformed_data.py
@@ -7,6 +7,21 @@ from altair import (
     HConcatChart,
     VConcatChart,
     ConcatChart,
+    TopLevelUnitSpec,
+    FacetedUnitSpec,
+    UnitSpec,
+    UnitSpecWithFrame,
+    NonNormalizedSpec,
+    TopLevelLayerSpec,
+    LayerSpec,
+    TopLevelConcatSpec,
+    ConcatSpecGenericSpec,
+    TopLevelHConcatSpec,
+    HConcatSpecGenericSpec,
+    TopLevelVConcatSpec,
+    VConcatSpecGenericSpec,
+    TopLevelFacetSpec,
+    FacetSpec,
     data_transformers,
 )
 from altair.utils._vegafusion_data import get_inline_tables
@@ -17,6 +32,25 @@ Scope = Tuple[int, ...]
 FacetMapping = Dict[Tuple[str, Scope], Tuple[str, Scope]]
 
 
+# For the transformed_data functionality, the chart classes in the values
+# can be considered equivalent to the chart class in the key.
+_chart_class_mapping = {
+    Chart: (
+        Chart,
+        TopLevelUnitSpec,
+        FacetedUnitSpec,
+        UnitSpec,
+        UnitSpecWithFrame,
+        NonNormalizedSpec,
+    ),
+    LayerChart: (LayerChart, TopLevelLayerSpec, LayerSpec),
+    ConcatChart: (ConcatChart, TopLevelConcatSpec, ConcatSpecGenericSpec),
+    HConcatChart: (HConcatChart, TopLevelHConcatSpec, HConcatSpecGenericSpec),
+    VConcatChart: (VConcatChart, TopLevelVConcatSpec, VConcatSpecGenericSpec),
+    FacetChart: (FacetChart, TopLevelFacetSpec, FacetSpec),
+}
+
+
 @overload
 def transformed_data(
     chart: Union[Chart, FacetChart],
@@ -118,6 +152,16 @@ def transformed_data(chart, row_limit=None, exclude=None):
         return datasets
 
 
+# The equivalent classes from _chart_class_mapping should also be added
+# to the type hints below for `chart` as the function would also work for them.
+# However, this was not possible so far as mypy then complains about
+# "Overloaded function signatures 1 and 2 overlap with incompatible return types [misc]"
+# This might be due to the complex type hierarchy of the chart classes.
+# See also https://github.com/python/mypy/issues/5119
+# and https://github.com/python/mypy/issues/4020 which show that mypy might not have
+# a very consistent behavior for overloaded functions.
+# The same error appeared when trying it with Protocols for the concat and layer charts.
+# This function is only used internally and so we accept this inconsistency for now.
 def name_views(
     chart: Union[
         Chart, FacetChart, LayerChart, HConcatChart, VConcatChart, ConcatChart
@@ -148,7 +192,9 @@ def name_views(
         List of the names of the charts and subcharts
     """
     exclude = set(exclude) if exclude is not None else set()
-    if isinstance(chart, (Chart, FacetChart)):
+    if isinstance(chart, _chart_class_mapping[Chart]) or isinstance(
+        chart, _chart_class_mapping[FacetChart]
+    ):
         if chart.name not in exclude:
             if chart.name in (None, Undefined):
                 # Add name since none is specified
@@ -157,13 +203,13 @@ def name_views(
         else:
             return []
     else:
-        if isinstance(chart, LayerChart):
+        if isinstance(chart, _chart_class_mapping[LayerChart]):
             subcharts = chart.layer
-        elif isinstance(chart, HConcatChart):
+        elif isinstance(chart, _chart_class_mapping[HConcatChart]):
             subcharts = chart.hconcat
-        elif isinstance(chart, VConcatChart):
+        elif isinstance(chart, _chart_class_mapping[VConcatChart]):
             subcharts = chart.vconcat
-        elif isinstance(chart, ConcatChart):
+        elif isinstance(chart, _chart_class_mapping[ConcatChart]):
             subcharts = chart.concat
         else:
             raise ValueError(
diff --git a/altair/utils/_vegafusion_data.py b/altair/utils/_vegafusion_data.py
index a462cb81..dac39d8d 100644
--- a/altair/utils/_vegafusion_data.py
+++ b/altair/utils/_vegafusion_data.py
@@ -1,14 +1,10 @@
-import sys
 from toolz import curried
 import uuid
 from weakref import WeakValueDictionary
 
 from typing import Union, Dict, Set, MutableMapping
 
-if sys.version_info >= (3, 8):
-    from typing import TypedDict, Final
-else:
-    from typing_extensions import TypedDict, Final
+from typing import TypedDict, Final
 
 from altair.utils.core import _DataFrameLike
 from altair.utils.data import _DataType, _ToValuesReturnType, MaxRowsError
diff --git a/altair/utils/core.py b/altair/utils/core.py
index c429a898..4a375d25 100644
--- a/altair/utils/core.py
+++ b/altair/utils/core.py
@@ -23,10 +23,7 @@ if sys.version_info >= (3, 10):
 else:
     from typing_extensions import ParamSpec
 
-if sys.version_info >= (3, 8):
-    from typing import Literal, Protocol
-else:
-    from typing_extensions import Literal, Protocol
+from typing import Literal, Protocol
 
 try:
     from pandas.api.types import infer_dtype as _infer_dtype
diff --git a/altair/utils/data.py b/altair/utils/data.py
index 9355dbef..e11a3655 100644
--- a/altair/utils/data.py
+++ b/altair/utils/data.py
@@ -2,7 +2,6 @@ import json
 import os
 import random
 import hashlib
-import sys
 import warnings
 from typing import Union, MutableMapping, Optional, Dict, Sequence, TYPE_CHECKING, List
 from types import ModuleType
@@ -17,10 +16,7 @@ from .deprecation import AltairDeprecationWarning
 from .plugin_registry import PluginRegistry
 
 
-if sys.version_info >= (3, 8):
-    from typing import Protocol, TypedDict, Literal
-else:
-    from typing_extensions import Protocol, TypedDict, Literal
+from typing import Protocol, TypedDict, Literal
 
 
 if TYPE_CHECKING:
diff --git a/altair/utils/execeval.py b/altair/utils/execeval.py
index 514f874c..7e98fd37 100644
--- a/altair/utils/execeval.py
+++ b/altair/utils/execeval.py
@@ -2,14 +2,6 @@ import ast
 import sys
 
 
-if sys.version_info > (3, 8):
-    Module = ast.Module
-else:
-    # Mock the Python >= 3.8 API
-    def Module(nodelist, type_ignores):
-        return ast.Module(nodelist)
-
-
 class _CatchDisplay:
     """Class to temporarily catch sys.displayhook"""
 
@@ -48,7 +40,7 @@ def eval_block(code, namespace=None, filename="<string>"):
         to_exec, to_eval = tree.body, []
 
     for node in to_exec:
-        compiled = compile(Module([node], []), filename=filename, mode="exec")
+        compiled = compile(ast.Module([node], []), filename=filename, mode="exec")
         exec(compiled, namespace)
 
     with catch_display:
diff --git a/altair/utils/plugin_registry.py b/altair/utils/plugin_registry.py
index 37d3db22..f6281ed1 100644
--- a/altair/utils/plugin_registry.py
+++ b/altair/utils/plugin_registry.py
@@ -1,11 +1,7 @@
-import sys
 from typing import Any, Dict, List, Optional, Generic, TypeVar, cast
 from types import TracebackType
 
-if sys.version_info >= (3, 8):
-    from importlib.metadata import entry_points
-else:
-    from importlib_metadata import entry_points
+from importlib.metadata import entry_points
 
 from toolz import curry
 
diff --git a/altair/vegalite/v5/compiler.py b/altair/vegalite/v5/compiler.py
index a4e02f79..dd7971ec 100644
--- a/altair/vegalite/v5/compiler.py
+++ b/altair/vegalite/v5/compiler.py
@@ -1,11 +1,6 @@
-import sys
-
 from ...utils.compiler import VegaLiteCompilerRegistry
 
-if sys.version_info >= (3, 8):
-    from typing import Final
-else:
-    from typing_extensions import Final
+from typing import Final
 
 
 ENTRY_POINT_GROUP: Final = "altair.vegalite.v5.vegalite_compiler"
diff --git a/altair/vegalite/v5/data.py b/altair/vegalite/v5/data.py
index d9edc285..1e47db52 100644
--- a/altair/vegalite/v5/data.py
+++ b/altair/vegalite/v5/data.py
@@ -1,5 +1,3 @@
-import sys
-
 from ..data import (
     MaxRowsError,
     curry,
@@ -15,10 +13,7 @@ from ..data import (
 
 from ...utils._vegafusion_data import vegafusion_data_transformer
 
-if sys.version_info >= (3, 8):
-    from typing import Final
-else:
-    from typing_extensions import Final
+from typing import Final
 
 
 # ==============================================================================
diff --git a/altair/vegalite/v5/display.py b/altair/vegalite/v5/display.py
index 50e70108..714e17b4 100644
--- a/altair/vegalite/v5/display.py
+++ b/altair/vegalite/v5/display.py
@@ -1,5 +1,4 @@
 import os
-import sys
 from typing import Dict
 
 from ...utils.mimebundle import spec_to_mimebundle
@@ -14,10 +13,7 @@ from ..display import (
 
 from .schema import SCHEMA_VERSION
 
-if sys.version_info >= (3, 8):
-    from typing import Final
-else:
-    from typing_extensions import Final
+from typing import Final
 
 VEGALITE_VERSION: Final = SCHEMA_VERSION.lstrip("v")
 VEGA_VERSION: Final = "5"
diff --git a/altair/vegalite/v5/schema/channels.py b/altair/vegalite/v5/schema/channels.py
index 07f9f43e..20ffd54f 100644
--- a/altair/vegalite/v5/schema/channels.py
+++ b/altair/vegalite/v5/schema/channels.py
@@ -8,10 +8,7 @@ from altair.utils.schemapi import Undefined, with_property_setters
 from altair.utils import parse_shorthand
 from typing import overload, List
 
-if sys.version_info >= (3, 8):
-    from typing import Literal
-else:
-    from typing_extensions import Literal
+from typing import Literal
 
 
 class FieldChannelMixin:
diff --git a/altair/vegalite/v5/theme.py b/altair/vegalite/v5/theme.py
index cdaca2db..262a8c9a 100644
--- a/altair/vegalite/v5/theme.py
+++ b/altair/vegalite/v5/theme.py
@@ -1,6 +1,5 @@
 """Tools for enabling and registering chart themes"""
-import sys
-from typing import Dict, Union
+from typing import Dict, Union, Final
 
 from ...utils.theme import ThemeRegistry
 
@@ -17,11 +16,6 @@ VEGA_THEMES = [
     "powerbi",
 ]
 
-if sys.version_info >= (3, 8):
-    from typing import Final
-else:
-    from typing_extensions import Final
-
 
 class VegaTheme:
     """Implementation of a builtin vega theme."""
diff --git a/doc/releases/changes.rst b/doc/releases/changes.rst
index 3f33bb28..b98d01a1 100644
--- a/doc/releases/changes.rst
+++ b/doc/releases/changes.rst
@@ -14,6 +14,7 @@ Bug Fixes
 
 Backward-Incompatible Changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- Drop support for Python 3.7 which is end-of-life (#3100).
 
 Version 5.0.1 (released May 26, 2023)
 -------------------------------------
diff --git a/pyproject.toml b/pyproject.toml
index baf6730c..c3da1bfd 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -16,7 +16,6 @@ build-backend = "hatchling.build"
 name = "altair"
 authors = [ {name = "Vega-Altair Contributors"} ]
 dependencies = [
-    "importlib_metadata; python_version<\"3.8\"",
     "typing_extensions>=4.0.1; python_version<\"3.11\"",
     "jinja2",
     "jsonschema>=3.0",
@@ -34,7 +33,7 @@ keywords = [
     "json",
     "vega-lite",
 ]
-requires-python = ">=3.7"
+requires-python = ">=3.8"
 dynamic = ["version"]
 license-files = { paths = ["LICENSE"] }
 classifiers= [
@@ -43,7 +42,6 @@ classifiers= [
     "Intended Audience :: Science/Research",
     "License :: OSI Approved :: BSD License",
     "Natural Language :: English",
-    "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
@@ -153,7 +151,7 @@ publish-clean-build = [
 
 [tool.black]
 line-length = 88
-target-version = ["py37", "py38", "py39", "py310", "py311"]
+target-version = ["py38", "py39", "py310", "py311"]
 include = '\.pyi?$'
 extend-exclude = '''
 /(
@@ -170,7 +168,7 @@ extend-exclude = '''
 '''
 
 [tool.ruff]
-target-version = "py310"
+target-version = "py38"
 line-length = 88
 select = [
     # flake8-bugbear
diff --git a/tools/generate_schema_wrapper.py b/tools/generate_schema_wrapper.py
index 7317495b..b6104f0b 100644
--- a/tools/generate_schema_wrapper.py
+++ b/tools/generate_schema_wrapper.py
@@ -424,10 +424,7 @@ def generate_vegalite_channel_wrappers(schemafile, version, imports=None):
             "from altair.utils import parse_shorthand",
             "from typing import overload, List",
             "",
-            "if sys.version_info >= (3, 8):",
-            "    from typing import Literal",
-            "else:",
-            "    from typing_extensions import Literal",
+            "from typing import Literal",
         ]
     contents = [HEADER]
     contents.extend(imports)
diff --git a/tools/update_init_file.py b/tools/update_init_file.py
index 712d3f57..f4c1f11e 100644
--- a/tools/update_init_file.py
+++ b/tools/update_init_file.py
@@ -15,10 +15,7 @@ if sys.version_info >= (3, 11):
 else:
     from typing_extensions import Self
 
-if sys.version_info >= (3, 8):
-    from typing import Literal
-else:
-    from typing_extensions import Literal
+from typing import Literal
 
 # Import Altair from head
 ROOT_DIR = abspath(join(dirname(__file__), ".."))


Test patch:
diff --git a/tests/test_transformed_data.py b/tests/test_transformed_data.py
index 928b539f..709961a3 100644
--- a/tests/test_transformed_data.py
+++ b/tests/test_transformed_data.py
@@ -57,10 +57,17 @@ import pytest
     ("window_rank.py", 12, ["team", "diff"]),
 ])
 # fmt: on
-def test_primitive_chart_examples(filename, rows, cols):
+@pytest.mark.parametrize("to_reconstruct", [True, False])
+def test_primitive_chart_examples(filename, rows, cols, to_reconstruct):
     source = pkgutil.get_data(examples_methods_syntax.__name__, filename)
     chart = eval_block(source)
+    if to_reconstruct:
+        # When reconstructing a Chart, Altair uses different classes
+        # then what might have been originally used. See
+        # https://github.com/hex-inc/vegafusion/issues/354 for more info.
+        chart = alt.Chart.from_dict(chart.to_dict())
     df = chart.transformed_data()
+
     assert len(df) == rows
     assert set(cols).issubset(set(df.columns))
 
@@ -96,19 +103,29 @@ def test_primitive_chart_examples(filename, rows, cols):
     ("us_population_pyramid_over_time.py", [19, 38, 19], [["gender"], ["year"], ["gender"]]),
 ])
 # fmt: on
-def test_compound_chart_examples(filename, all_rows, all_cols):
+@pytest.mark.parametrize("to_reconstruct", [True, False])
+def test_compound_chart_examples(filename, all_rows, all_cols, to_reconstruct):
     source = pkgutil.get_data(examples_methods_syntax.__name__, filename)
     chart = eval_block(source)
-    print(chart)
-
+    if to_reconstruct:
+        # When reconstructing a Chart, Altair uses different classes
+        # then what might have been originally used. See
+        # https://github.com/hex-inc/vegafusion/issues/354 for more info.
+        chart = alt.Chart.from_dict(chart.to_dict())
     dfs = chart.transformed_data()
-    assert len(dfs) == len(all_rows)
-    for df, rows, cols in zip(dfs, all_rows, all_cols):
-        assert len(df) == rows
-        assert set(cols).issubset(set(df.columns))
+
+    if not to_reconstruct:
+        # Only run assert statements if the chart is not reconstructed. Reason
+        # is that for some charts, the original chart contained duplicated datasets
+        # which disappear when reconstructing the chart.
+        assert len(dfs) == len(all_rows)
+        for df, rows, cols in zip(dfs, all_rows, all_cols):
+            assert len(df) == rows
+            assert set(cols).issubset(set(df.columns))
 
 
-def test_transformed_data_exclude():
+@pytest.mark.parametrize("to_reconstruct", [True, False])
+def test_transformed_data_exclude(to_reconstruct):
     source = data.wheat()
     bar = alt.Chart(source).mark_bar().encode(x="year:O", y="wheat:Q")
     rule = alt.Chart(source).mark_rule(color="red").encode(y="mean(wheat):Q")
@@ -119,6 +136,11 @@ def test_transformed_data_exclude():
     )
 
     chart = (bar + rule + some_annotation).properties(width=600)
+    if to_reconstruct:
+        # When reconstructing a Chart, Altair uses different classes
+        # then what might have been originally used. See
+        # https://github.com/hex-inc/vegafusion/issues/354 for more info.
+        chart = alt.Chart.from_dict(chart.to_dict())
     datasets = chart.transformed_data(exclude=["some_annotation"])
 
     assert len(datasets) == 2