Repository: napari/napari

Issue:
Playback loop mode serializing issue.
## 🐛 Bug

Found by @ppwadhwa in https://github.com/napari/napari/pull/3123#issuecomment-892317829, I can't get the TB but can reproduce a similar issue.

Changing the Playback loop mode erases the settings :

https://user-images.githubusercontent.com/335567/129407915-44630d10-ec51-4189-9cb9-e599f2b9a93b.mov


## To Reproduce

- Open napari settings file
- Open napari
- Go to settings, Application Setting, Playback loop mode. 
- Change Loop Mode. 
- recheck settings file, it is now empty.

## Expected behavior

1) don't erase setting
2) get an error message if something is wrong.


Discussion:

Resolving patch:
diff --git a/napari/settings/_base.py b/napari/settings/_base.py
--- a/napari/settings/_base.py
+++ b/napari/settings/_base.py
@@ -14,6 +14,7 @@
 from ..utils.events import EmitterGroup, EventedModel
 from ..utils.misc import deep_update
 from ..utils.translations import trans
+from ._yaml import PydanticYamlMixin
 
 _logger = logging.getLogger(__name__)
 
@@ -59,7 +60,7 @@ def _on_sub_event(self, event: Event, field=None):
 _NOT_SET = object()
 
 
-class EventedConfigFileSettings(EventedSettings):
+class EventedConfigFileSettings(EventedSettings, PydanticYamlMixin):
     """This adds config read/write and yaml support to EventedSettings.
 
     If your settings class *only* needs to read variables from the environment,
@@ -69,6 +70,9 @@ class EventedConfigFileSettings(EventedSettings):
 
     _config_path: Optional[Path] = None
     _save_on_change: bool = True
+    # this dict stores the data that came specifically from the config file.
+    # it's populated in `config_file_settings_source` and
+    # used in `_remove_env_settings`
     _config_file_settings: dict
 
     # provide config_path=None to prevent reading from disk.
@@ -92,9 +96,10 @@ def _on_sub_event(self, event, field=None):
 
     @property
     def config_path(self):
+        """Return the path to/from which settings be saved/loaded."""
         return self._config_path
 
-    def dict(  # type: ignore
+    def dict(
         self,
         *,
         include: Union[AbstractSetIntStr, MappingIntStrAny] = None,  # type: ignore
@@ -105,6 +110,10 @@ def dict(  # type: ignore
         exclude_none: bool = False,
         exclude_env: bool = False,
     ) -> DictStrAny:
+        """Return dict representation of the model.
+
+        May optionally specify which fields to include or exclude.
+        """
         data = super().dict(
             include=include,
             exclude=exclude,
@@ -114,64 +123,50 @@ def dict(  # type: ignore
             exclude_none=exclude_none,
         )
         if exclude_env:
-            eset = getattr(self.__config__, '_env_settings', None)
-            if callable(eset):
-                env_data: dict = eset(type(self))
-                if env_data:
-                    cfg_data = getattr(self, '_config_file_settings', {})
-                    _restore_config_data(data, env_data, cfg_data)
+            self._remove_env_settings(data)
         return data
 
-    def yaml(
-        self,
-        *,
-        include: Union[AbstractSetIntStr, MappingIntStrAny] = None,  # type: ignore
-        exclude: Union[AbstractSetIntStr, MappingIntStrAny] = None,  # type: ignore
-        by_alias: bool = False,
-        exclude_unset: bool = False,
-        exclude_defaults: bool = False,
-        exclude_none: bool = False,
-        exclude_env: bool = False,
-        **dumps_kwargs: Any,
-    ) -> str:
-        import json
-
-        import yaml
+    def _save_dict(self, **dict_kwargs):
+        """The minimal dict representation that will be persisted to disk.
 
-        data = self.dict(
-            include=include,
-            exclude=exclude,
-            by_alias=by_alias,
-            exclude_unset=exclude_unset,
-            exclude_defaults=exclude_defaults,
-            exclude_none=exclude_none,
-            exclude_env=exclude_env,
-        )
+        By default, this will exclude settings values that match the default
+        value, and will exclude values that were provided by environment
+        variables.  Empty dicts will also be removed.
+        """
+        dict_kwargs.setdefault('exclude_defaults', True)
+        dict_kwargs.setdefault('exclude_env', True)
+        data = self.dict(**dict_kwargs)
         _remove_empty_dicts(data)
+        return data
 
-        # We roundtrip to keep custom string objects (like SchemaVersion)
-        # yaml representable
-        # FIXME: should provide yaml serializer on field itself (as for json)
-        _json = self.__config__.json_dumps(data, default=self.__json_encoder__)
-        return yaml.safe_dump(json.loads(_json), **dumps_kwargs)
+    def save(self, path: Union[str, Path] = None, **dict_kwargs):
+        """Save current settings to path.
 
-    def save(self, path=None, **dict_kwargs):
+        By default, this will exclude settings values that match the default
+        value, and will exclude values that were provided by environment
+        variables.  (see `_save_dict` method.)
+        """
         path = path or self.config_path
         if not path:
             raise ValueError("No path provided in config or save argument.")
 
         path = Path(path).expanduser().resolve()
         path.parent.mkdir(exist_ok=True, parents=True)
-
-        dict_kwargs.setdefault('exclude_defaults', True)
-        dict_kwargs.setdefault('exclude_env', True)
-        data = self.dict(**dict_kwargs)
-        _remove_empty_dicts(data)
-
-        dump = _get_io_func_for_path(path, 'dump')
-        if dump is not None:
-            with open(path, 'w') as target:
-                dump(data, target)
+        self._dump(path, self._save_dict(**dict_kwargs))
+
+    def _dump(self, path: str, data: dict):
+        """Encode and dump `data` to `path` using a path-appropriate encoder."""
+        if str(path).endswith(('.yaml', '.yml')):
+            data = self._yaml_dump(data)
+        elif str(path).endswith(".json"):
+            json_dumps = self.__config__.json_dumps
+            data = json_dumps(data, default=self.__json_encoder__)
+        else:
+            raise NotImplementedError(
+                f"Can only currently dump to `.json` or `.yaml`, not {path!r}"
+            )
+        with open(path, 'w') as target:
+            target.write(data)
 
     def env_settings(self) -> Dict[str, Any]:
         """Get a dict of fields that were provided as environment vars."""
@@ -180,6 +175,19 @@ def env_settings(self) -> Dict[str, Any]:
             env_settings = env_settings(self)
         return env_settings
 
+    def _remove_env_settings(self, data):
+        """Remove key:values from `data` that match settings from env vars.
+
+        This is handy when we want to persist settings to disk without
+        including settings that were provided by environment variables (which
+        are usually more temporary).
+        """
+        env_data = self.env_settings()
+        if env_data:
+            _restore_config_data(
+                data, env_data, getattr(self, '_config_file_settings', {})
+            )
+
     class Config:
         # If True: validation errors in a config file will raise an exception
         # otherwise they will warn to the logger
@@ -279,7 +287,9 @@ def _inner(settings: BaseSettings) -> Dict[str, Any]:
     return _inner
 
 
-def config_file_settings_source(settings: BaseSettings) -> Dict[str, Any]:
+def config_file_settings_source(
+    settings: EventedConfigFileSettings,
+) -> Dict[str, Any]:
     """Read config files during init of an EventedConfigFileSettings obj.
 
     The two important values are the `settings._config_path`
@@ -331,8 +341,17 @@ def config_file_settings_source(settings: BaseSettings) -> Dict[str, Any]:
             continue
 
         # get loader for yaml/json
-        load = _get_io_func_for_path(_path, 'load')
-        if load is None:
+        if str(path).endswith(('.yaml', '.yml')):
+            load = __import__('yaml').safe_load
+        elif str(path).endswith(".json"):
+            load = __import__('json').load
+        else:
+            warnings.warn(
+                trans._(
+                    "Unrecognized file extension for config_path: {path}",
+                    path=path,
+                )
+            )
             continue
 
         try:
@@ -383,7 +402,8 @@ def config_file_settings_source(settings: BaseSettings) -> Dict[str, Any]:
                 )
             )
             data = {}
-    setattr(settings, '_config_file_settings', data)
+    # store data at this state for potential later recovery
+    settings._config_file_settings = data
     return data
 
 
@@ -449,19 +469,3 @@ def _remove_empty_dicts(dct: dict, recurse=True) -> dict:
         if v == {}:
             del dct[k]
     return dct
-
-
-def _get_io_func_for_path(path: Union[str, Path], mode='dump'):
-    """get json/yaml [safe_]load/[safe_]dump for `path`"""
-    assert mode in {'dump', 'load'}
-    if str(path).endswith(('.yaml', '.yml')):
-        return getattr(__import__('yaml'), f'safe_{mode}')
-    if str(path).endswith(".json"):
-        return getattr(__import__('json'), mode)
-
-    warnings.warn(
-        trans._(
-            "Unrecognized file extension for config_path: {path}", path=path
-        )
-    )
-    return None
diff --git a/napari/settings/_napari_settings.py b/napari/settings/_napari_settings.py
--- a/napari/settings/_napari_settings.py
+++ b/napari/settings/_napari_settings.py
@@ -13,7 +13,8 @@
 from ._plugins import PluginsSettings
 from ._shortcuts import ShortcutsSettings
 
-_CFG_PATH = os.getenv('NAPARI_CONFIG', _DEFAULT_CONFIG_PATH)
+_DEFAULT_CFG_PATH = os.getenv('NAPARI_CONFIG', _DEFAULT_CONFIG_PATH)
+_DEFAULT_CFG_PATH = Path(_DEFAULT_CFG_PATH) if _DEFAULT_CFG_PATH else None
 
 
 class NapariSettings(EventedConfigFileSettings):
@@ -46,7 +47,7 @@ class NapariSettings(EventedConfigFileSettings):
     )
 
     # private attributes and ClassVars will not appear in the schema
-    _config_path: Optional[Path] = Path(_CFG_PATH)
+    _config_path: Optional[Path] = _DEFAULT_CFG_PATH
 
     class Config:
         env_prefix = 'napari_'
@@ -55,7 +56,7 @@ class Config:
     def __str__(self):
         out = 'NapariSettings (defaults excluded)\n'
         out += '----------------------------------\n'
-        out += self.yaml(exclude_defaults=True)
+        out += self._yaml_dump(self._save_dict())
         return out
 
     def __repr__(self):
diff --git a/napari/settings/_yaml.py b/napari/settings/_yaml.py
new file mode 100644
--- /dev/null
+++ b/napari/settings/_yaml.py
@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel
+from yaml import Dumper, SafeDumper, dump_all
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+    from typing import AbstractSet, Any, Dict, Optional, TypeVar, Union
+
+    IntStr = Union[int, str]
+    AbstractSetIntStr = AbstractSet[IntStr]
+    DictStrAny = Dict[str, Any]
+    MappingIntStrAny = Mapping[IntStr, Any]
+    Model = TypeVar('Model', bound=BaseModel)
+
+
+class YamlDumper(SafeDumper):
+    """The default YAML serializer for our pydantic models.
+
+    Add support for custom types by using `YamlDumper.add_representer` or
+    `YamlDumper.add_multi_representer` below.
+    """
+
+
+# add_representer requires a strict type match
+# add_multi_representer also works for all subclasses of the provided type.
+YamlDumper.add_multi_representer(str, YamlDumper.represent_str)
+YamlDumper.add_multi_representer(
+    Enum, lambda dumper, data: dumper.represent_str(data.value)
+)
+# the default set representer is ugly:
+# disabled_plugins: !!set
+#   bioformats: null
+# and pydantic will make sure that incoming sets are converted to sets
+YamlDumper.add_representer(
+    set, lambda dumper, data: dumper.represent_list(data)
+)
+
+
+class PydanticYamlMixin(BaseModel):
+    """Mixin that provides yaml dumping capability to pydantic BaseModel.
+
+    To provide a custom yaml Dumper on a subclass, provide a `yaml_dumper`
+    on the Config:
+
+        class Config:
+            yaml_dumper = MyDumper
+    """
+
+    def yaml(
+        self,
+        *,
+        include: Union[AbstractSetIntStr, MappingIntStrAny] = None,  # type: ignore
+        exclude: Union[AbstractSetIntStr, MappingIntStrAny] = None,  # type: ignore
+        by_alias: bool = False,
+        exclude_unset: bool = False,
+        exclude_defaults: bool = False,
+        exclude_none: bool = False,
+        dumper: Optional[Dumper] = None,
+        **dumps_kwargs: Any,
+    ) -> str:
+        """Serialize model to yaml."""
+        data = self.dict(
+            include=include,
+            exclude=exclude,
+            by_alias=by_alias,
+            exclude_unset=exclude_unset,
+            exclude_defaults=exclude_defaults,
+            exclude_none=exclude_none,
+        )
+        if self.__custom_root_type__:
+            from pydantic.utils import ROOT_KEY
+
+            data = data[ROOT_KEY]
+        return self._yaml_dump(data, dumper, **dumps_kwargs)
+
+    def _yaml_dump(self, data, dumper: Optional[Dumper] = None, **kw):
+        dumper = dumper or getattr(self.__config__, 'yaml_dumper', YamlDumper)
+        return dump_all([data], Dumper=dumper, **kw)


Test patch:
diff --git a/napari/settings/_tests/test_settings.py b/napari/settings/_tests/test_settings.py
--- a/napari/settings/_tests/test_settings.py
+++ b/napari/settings/_tests/test_settings.py
@@ -228,20 +228,30 @@ class T(NapariSettings):
 # Failing because dark is actually the default...
 def test_settings_env_variables_do_not_write_to_disk(tmp_path, monkeypatch):
 
+    # create a settings file with light theme
     data = "appearance:\n   theme: light"
     fake_path = tmp_path / 'fake_path.yml'
     fake_path.write_text(data)
 
+    # make sure they wrote correctly
     disk_settings = fake_path.read_text()
     assert 'theme: light' in disk_settings
+    # make sure they load correctly
     assert NapariSettings(fake_path).appearance.theme == "light"
 
+    # now load settings again with an Env-var override
     monkeypatch.setenv('NAPARI_APPEARANCE_THEME', 'dark')
     settings = NapariSettings(fake_path)
+    # make sure the override worked, and save again
     assert settings.appearance.theme == 'dark'
-    settings.save()
+    # data from the config file is still "known"
+    assert settings._config_file_settings['appearance']['theme'] == 'light'
+    # but we know what came from env vars as well:
+    assert settings.env_settings()['appearance']['theme'] == 'dark'
 
-    # it shouldn't have overriden our non-default value
+    # when we save it shouldn't use environment variables and it shouldn't
+    # have overriden our non-default value of `theme: light`
+    settings.save()
     disk_settings = fake_path.read_text()
     assert 'theme: light' in disk_settings
 
@@ -323,3 +333,12 @@ def test_settings_events(test_settings):
     mock.reset_mock()
     test_settings.appearance.theme = 'light'
     mock.assert_not_called()
+
+
+@pytest.mark.parametrize('ext', ['yml', 'yaml', 'json'])
+def test_full_serialize(test_settings: NapariSettings, tmp_path, ext):
+    """Make sure that every object in the settings is serializeable.
+
+    Should work with both json and yaml.
+    """
+    test_settings.save(tmp_path / f't.{ext}', exclude_defaults=False)