Repository: pybamm-team/PyBaMM (python)

Issue:
Tag PyBaMM citations
**Summary**
The way citations are printed now (all together) can make it quite hard for an inexperienced user to know why the need to cite each reference. It would be good to add a tag when registering PyBaMM citations and also some extra functionality that, when printing the citations, reads the tags and groups them in a more meaningful way. For example, we could print some sort of list or table that lists:

- Model: Fuller1994
- Parameters: Chen2020
- Solver: Andersson 2019
and so on...

Please add your comments and suggestions

 **Motivation**
 Make it easier for new or inexperienced users to know what's the purpose of each reference, which might lead in them just dumping the citations in their papers all together. Adding this functionality should lead to higher quality citations in the papers.


Pull request:
Issue 1182 faster tagging for PyBaMM citations 
# Description

Enables a faster method to tag citations based on where they come up in the code during a simulation (models, solvers, etc.) – via a `caller_name` method that calls `sys._getframe()` to access current frames for the `pybamm.citations.register()` function. 

This PR continues the discussion on #2862 and #2908, with some results of the speedup available at https://github.com/pybamm-team/PyBaMM/issues/2908#issuecomment-1537077212. Here, `sys._getframe` is preferred to `inspect._getframe()` for avoiding the internal `hasattr()`, in order to reduce any potential performance impact(s) of attribute checks at runtime.

Fixes #1182 

## Type of change

Please add a line in the relevant section of [CHANGELOG.md](https://github.com/pybamm-team/PyBaMM/blob/develop/CHANGELOG.md) to document the change (include PR #) - note reverse order of PR #s. If necessary, also add to the list of breaking changes.

- [x] New feature (non-breaking change which adds functionality)
- [ ] Optimization (back-end change that speeds up the code)
- [ ] Bug fix (non-breaking change which fixes an issue)

# Key checklist:

- [x] No style issues: `$ pre-commit run` (see [CONTRIBUTING.md](https://github.com/pybamm-team/PyBaMM/blob/develop/CONTRIBUTING.md#installing-and-using-pre-commit) for how to set this up to run automatically when committing locally, in just two lines of code)
- [x] All tests pass: `$ python run-tests.py --all`
- [x] The documentation builds: `$ python run-tests.py --doctest`

You can run unit and doctests together at once, using `$ python run-tests.py --quick`.

## Further checks:

- [x] Code is commented, particularly in hard-to-understand areas
- [x] Tests added that prove fix is effective or that feature works


Patch:
diff --git a/pybamm/citations.py b/pybamm/citations.py
index f236da8d7..d01ccc505 100644
--- a/pybamm/citations.py
+++ b/pybamm/citations.py
@@ -7,6 +7,7 @@ import pybamm
 import os
 import warnings
 import pybtex
+from sys import _getframe
 from pybtex.database import parse_file, parse_string, Entry
 from pybtex.scanner import PybtexError
 
@@ -35,6 +36,12 @@ class Citations:
         # Dict mapping citations keys to BibTex entries
         self._all_citations: dict[str, str] = dict()
 
+        # Set of unknown citations to parse with pybtex
+        self._unknown_citations = set()
+
+        # Dict mapping citation tags for use when registering citations
+        self._citation_tags = dict()
+
         # store citation error
         self._citation_err_msg = None
 
@@ -48,10 +55,23 @@ class Citations:
         """Reset citations to default only (only for testing purposes)"""
         # Initialize empty papers to cite
         self._papers_to_cite = set()
-        # Register the PyBaMM paper and the numpy paper
+        # Initialize empty set of unknown citations
+        self._unknown_citations = set()
+        # Initialize empty citation tags
+        self._citation_tags = dict()
+        # Register the PyBaMM paper and the NumPy paper
         self.register("Sulzer2021")
         self.register("Harris2020")
 
+    def _caller_name():
+        """
+        Returns the qualified name of classes that call :meth:`register` internally.
+        Gets cached in order to reduce the number of calls.
+        """
+        # Attributed to https://stackoverflow.com/a/53490973
+        caller_name = _getframe().f_back.f_back.f_locals["self"].__class__.__qualname__
+        return caller_name
+
     def read_citations(self):
         """Reads the citations in `pybamm.CITATIONS.txt`. Other works can be cited
         by passing a BibTeX citation to :meth:`register`.
@@ -70,7 +90,7 @@ class Citations:
         if not isinstance(key, str) or not isinstance(entry, Entry):
             raise TypeError()
 
-        # Warn if overwriting an previous citation
+        # Warn if overwriting a previous citation
         new_citation = entry.to_string("bibtex")
         if key in self._all_citations and new_citation != self._all_citations[key]:
             warnings.warn(f"Replacing citation for {key}")
@@ -78,14 +98,20 @@ class Citations:
         # Add to database
         self._all_citations[key] = new_citation
 
+    def _add_citation_tag(self, key, entry):
+        """Adds a tag for a citation key in the dict, which represents the name of the
+        class that called :meth:`register`"""
+        self._citation_tags[key] = entry
+
     @property
     def _cited(self):
         """Return a list of the BibTeX entries that have been cited"""
         return [self._all_citations[key] for key in self._papers_to_cite]
 
     def register(self, key):
-        """Register a paper to be cited. The intended use is that :meth:`register`
-        should be called only when the referenced functionality is actually being used.
+        """Register a paper to be cited, one at a time. The intended use is that
+        :meth:`register` should be called only when the referenced functionality is
+        actually being used.
 
         .. warning::
             Registering a BibTeX citation, with the same key as an existing citation,
@@ -95,39 +121,113 @@ class Citations:
         ----------
         key : str
             - The citation key for an entry in `pybamm/CITATIONS.txt` or
-            - One or more BibTeX formatted citations
+            - A BibTeX formatted citation
         """
         if self._citation_err_msg is None:
             # Check if citation is a known key
             if key in self._all_citations:
+                self._papers_to_cite.add(key)
+                # Add citation tags for the key for verbose output, but
+                # don't if they already exist in _citation_tags dict
+                if key not in self._citation_tags:
+                    try:
+                        caller = Citations._caller_name()
+                        self._add_citation_tag(key, entry=caller)
+                        # Don't add citation tags if the citation is registered manually
+                    except KeyError:  # pragma: no cover
+                        pass
+            else:
+                # If citation is unknown, parse it later with pybtex
+                self._unknown_citations.add(key)
+                return
+
+    def _parse_citation(self, key):
+        """
+        Parses a citation with pybtex and adds it to the _papers_to_cite set. This
+        method is called when a citation is unknown at the time of registration.
+
+        Parameters
+        ----------
+        key: str
+            A BibTeX formatted citation
+        """
+        try:
+            # Parse string as a bibtex citation, and check that a citation was found
+            bib_data = parse_string(key, bib_format="bibtex")
+            if not bib_data.entries:
+                raise PybtexError("no entries found")
+
+            # Add and register all citations
+            for key, entry in bib_data.entries.items():
+                # Add to _all_citations dictionary
+                self._add_citation(key, entry)
+                # Add to _papers_to_cite set
                 self._papers_to_cite.add(key)
                 return
+        except PybtexError:
+            # Unable to parse / unknown key
+            raise KeyError(f"Not a bibtex citation or known citation: {key}")
 
-            # Try to parse the citation using pybtex
-            try:
-                # Parse string as a bibtex citation, and check that a citation was found
-                bib_data = parse_string(key, bib_format="bibtex")
-                if not bib_data.entries:
-                    raise PybtexError("no entries found")
-
-                # Add and register all citations
-                for key, entry in bib_data.entries.items():
-                    self._add_citation(key, entry)
-                    self.register(key)
-                    return
-            except PybtexError:
-                # Unable to parse / unknown key
-                raise KeyError(f"Not a bibtex citation or known citation: {key}")
-
-    def print(self, filename=None, output_format="text"):
-        """Print all citations that were used for running simulations.
+    def _tag_citations(self):
+        """Prints the citation tags for the citations that have been registered
+        (non-manually) in the code, for verbose output purposes
+        """
+        if self._citation_tags:  # pragma: no cover
+            print("\nCitations registered: \n")
+            for key, entry in self._citation_tags.items():
+                print(f"{key} was cited due to the use of {entry}")
+
+    def print(self, filename=None, output_format="text", verbose=False):
+        """Print all citations that were used for running simulations. The verbose
+        option is provided to print tags for citations in the output such that it can
+        can be seen where the citations were registered due to the use of PyBaMM models
+        and solvers in the code.
+
+        .. note::
+            If a citation is registered manually, it will not be tagged.
+
+        .. warning::
+            This function will notify the user if a citation that has been previously
+            registered is invalid or cannot be parsed.
 
         Parameters
         ----------
         filename : str, optional
-            Filename to which to print citations. If None, citations are printed to the
-            terminal.
+            Filename to which to print citations. If None, citations are printed
+            to the terminal.
+        verbose: bool, optional
+            If True, prints the citation tags for the citations that have been
+            registered. An example of the output is shown below.
+
+        Examples
+        --------
+        .. code-block:: python
+
+            pybamm.lithium_ion.SPM()
+            pybamm.Citations.print(verbose=True) or pybamm.print_citations(verbose=True)
+
+        will append the following at the end of the list of citations:
+
+        .. code-block::
+
+            Citations registered:
+
+            Marquis2019 was cited due to the use of SPM
+
         """
+        # Parse citations that were not known keys at registration, but do not
+        # fail if they cannot be parsed
+        try:
+            for key in self._unknown_citations:
+                self._parse_citation(key)
+        except KeyError:  # pragma: no cover
+            warnings.warn(
+                message=f'\nCitation with key "{key}" is invalid. Please try again\n',
+                category=UserWarning,
+            )
+            # delete the invalid citation from the set
+            self._unknown_citations.remove(key)
+
         if output_format == "text":
             citations = pybtex.format_from_strings(
                 self._cited, style="plain", output_backend="plaintext"
@@ -142,12 +242,14 @@ class Citations:
 
         if filename is None:
             print(citations)
+            if verbose:
+                self._tag_citations()  # pragma: no cover
         else:
             with open(filename, "w") as f:
                 f.write(citations)
 
 
-def print_citations(filename=None, output_format="text"):
+def print_citations(filename=None, output_format="text", verbose=False):
     """See :meth:`Citations.print`"""
     if citations._citation_err_msg is not None:
         raise ImportError(
@@ -159,7 +261,15 @@ def print_citations(filename=None, output_format="text"):
             f"{citations._citation_err_msg}"
         )
     else:
-        pybamm.citations.print(filename, output_format)
+        if verbose:  # pragma: no cover
+            if filename is not None:  # pragma: no cover
+                raise Exception(
+                    "Verbose output is available only for the terminal and not for printing to files",  # noqa: E501
+                )
+            else:
+                citations.print(filename, output_format, verbose=True)
+        else:
+            pybamm.citations.print(filename, output_format)
 
 
 citations = Citations()


Test patch:
diff --git a/tests/unit/test_citations.py b/tests/unit/test_citations.py
index 5480182e6..d388f8d2b 100644
--- a/tests/unit/test_citations.py
+++ b/tests/unit/test_citations.py
@@ -13,7 +13,7 @@ from tempfile import NamedTemporaryFile
 
 @contextlib.contextmanager
 def temporary_filename():
-    """Create a temporary-file and return yield it's filename"""
+    """Create a temporary-file and return yield its filename"""
 
     f = NamedTemporaryFile(delete=False)
     try:
@@ -34,9 +34,15 @@ class TestCitations(unittest.TestCase):
         self.assertIn("Sulzer2019physical", citations._all_citations.keys())
         self.assertNotIn("Sulzer2019physical", citations._papers_to_cite)
 
-        # test key error
+        # Register a citation that does not exist
+        citations.register("not a citation")
+
+        # Test key error
         with self.assertRaises(KeyError):
-            citations.register("not a citation")
+            citations._parse_citation("not a citation")  # this should raise key error
+
+        # Test unknown citations at registration
+        self.assertIn("not a citation", citations._unknown_citations)
 
     def test_print_citations(self):
         pybamm.citations._reset()
@@ -69,22 +75,33 @@ class TestCitations(unittest.TestCase):
             pybamm.print_citations()
         pybamm.citations._citation_err_msg = None
 
+        # Test that unknown citation raises warning message on printing
+        pybamm.citations._reset()
+        pybamm.citations.register("not a citation")
+        with self.assertWarnsRegex(UserWarning, "not a citation"):
+            pybamm.print_citations()
+
     def test_overwrite_citation(self):
         # Unknown citation
         fake_citation = r"@article{NotACitation, title = {This Doesn't Exist}}"
         with warnings.catch_warnings():
             pybamm.citations.register(fake_citation)
+            pybamm.citations._parse_citation(fake_citation)
         self.assertIn("NotACitation", pybamm.citations._papers_to_cite)
 
         # Same NotACitation
         with warnings.catch_warnings():
             pybamm.citations.register(fake_citation)
+            pybamm.citations._parse_citation(fake_citation)
         self.assertIn("NotACitation", pybamm.citations._papers_to_cite)
 
         # Overwrite NotACitation
         old_citation = pybamm.citations._all_citations["NotACitation"]
         with self.assertWarns(Warning):
             pybamm.citations.register(r"@article{NotACitation, title = {A New Title}}")
+            pybamm.citations._parse_citation(
+                r"@article{NotACitation, title = {A New Title}}"
+            )  # noqa: E501
         self.assertIn("NotACitation", pybamm.citations._papers_to_cite)
         self.assertNotEqual(
             pybamm.citations._all_citations["NotACitation"], old_citation
@@ -92,8 +109,10 @@ class TestCitations(unittest.TestCase):
 
     def test_input_validation(self):
         """Test type validation of ``_add_citation``"""
+        pybamm.citations.register(1)
+
         with self.assertRaises(TypeError):
-            pybamm.citations.register(1)
+            pybamm.citations._parse_citation(1)
 
         with self.assertRaises(TypeError):
             pybamm.citations._add_citation("NotACitation", "NotAEntry")
@@ -116,6 +135,7 @@ class TestCitations(unittest.TestCase):
         self.assertNotIn("Marquis2019", citations._papers_to_cite)
         pybamm.lithium_ion.SPM(build=False)
         self.assertIn("Marquis2019", citations._papers_to_cite)
+        self.assertIn("Marquis2019", citations._citation_tags.keys())
 
         citations._reset()
         pybamm.lithium_ion.SPMe(build=False)
@@ -127,6 +147,7 @@ class TestCitations(unittest.TestCase):
         self.assertNotIn("Doyle1993", citations._papers_to_cite)
         pybamm.lithium_ion.DFN(build=False)
         self.assertIn("Doyle1993", citations._papers_to_cite)
+        self.assertIn("Doyle1993", citations._citation_tags.keys())
 
     def test_sulzer_2019(self):
         # Test that calling relevant bits of code adds the right paper to citations
@@ -136,10 +157,12 @@ class TestCitations(unittest.TestCase):
         self.assertNotIn("Sulzer2019asymptotic", citations._papers_to_cite)
         pybamm.lead_acid.LOQS(build=False)
         self.assertIn("Sulzer2019asymptotic", citations._papers_to_cite)
+        self.assertIn("Sulzer2019asymptotic", citations._citation_tags.keys())
 
         citations._reset()
         pybamm.lead_acid.Full(build=False)
         self.assertIn("Sulzer2019physical", citations._papers_to_cite)
+        self.assertIn("Sulzer2019physical", citations._citation_tags.keys())
 
     def test_timms_2021(self):
         # Test that calling relevant bits of code adds the right paper to citations
@@ -149,36 +172,43 @@ class TestCitations(unittest.TestCase):
         self.assertNotIn("Timms2021", citations._papers_to_cite)
         pybamm.current_collector.BasePotentialPair(param=None)
         self.assertIn("Timms2021", citations._papers_to_cite)
+        self.assertIn("Timms2021", citations._citation_tags.keys())
 
         citations._reset()
         self.assertNotIn("Timms2021", citations._papers_to_cite)
         pybamm.current_collector.EffectiveResistance()
         self.assertIn("Timms2021", citations._papers_to_cite)
+        self.assertIn("Timms2021", citations._citation_tags.keys())
 
         citations._reset()
         self.assertNotIn("Timms2021", citations._papers_to_cite)
         pybamm.current_collector.AlternativeEffectiveResistance2D()
         self.assertIn("Timms2021", citations._papers_to_cite)
+        self.assertIn("Timms2021", citations._citation_tags.keys())
 
         citations._reset()
         self.assertNotIn("Timms2021", citations._papers_to_cite)
         pybamm.thermal.pouch_cell.CurrentCollector1D(param=None)
         self.assertIn("Timms2021", citations._papers_to_cite)
+        self.assertIn("Timms2021", citations._citation_tags.keys())
 
         citations._reset()
         self.assertNotIn("Timms2021", citations._papers_to_cite)
         pybamm.thermal.pouch_cell.CurrentCollector2D(param=None)
         self.assertIn("Timms2021", citations._papers_to_cite)
+        self.assertIn("Timms2021", citations._citation_tags.keys())
 
         citations._reset()
         self.assertNotIn("Timms2021", citations._papers_to_cite)
         pybamm.thermal.Lumped(param=None)
         self.assertIn("Timms2021", citations._papers_to_cite)
+        self.assertIn("Timms2021", citations._citation_tags.keys())
 
         citations._reset()
         self.assertNotIn("Timms2021", citations._papers_to_cite)
         pybamm.thermal.OneDimensionalX(param=None)
         self.assertIn("Timms2021", citations._papers_to_cite)
+        self.assertIn("Timms2021", citations._citation_tags.keys())
 
     def test_subramanian_2005(self):
         # Test that calling relevant bits of code adds the right paper to citations
@@ -190,6 +220,7 @@ class TestCitations(unittest.TestCase):
             None, "negative", {"particle": "quadratic profile"}, "primary"
         )
         self.assertIn("Subramanian2005", citations._papers_to_cite)
+        self.assertIn("Subramanian2005", citations._citation_tags.keys())
 
         citations._reset()
         self.assertNotIn("Subramanian2005", citations._papers_to_cite)
@@ -197,6 +228,7 @@ class TestCitations(unittest.TestCase):
             None, "negative", {"particle": "quadratic profile"}, "primary"
         )
         self.assertIn("Subramanian2005", citations._papers_to_cite)
+        self.assertIn("Subramanian2005", citations._citation_tags.keys())
 
     def test_brosaplanella_2021(self):
         # Test that calling relevant bits of code adds the right paper to citations
@@ -206,6 +238,7 @@ class TestCitations(unittest.TestCase):
         self.assertNotIn("BrosaPlanella2021", citations._papers_to_cite)
         pybamm.electrolyte_conductivity.Integrated(None)
         self.assertIn("BrosaPlanella2021", citations._papers_to_cite)
+        self.assertIn("BrosaPlanella2021", citations._citation_tags.keys())
 
     def test_brosaplanella_2022(self):
         # Test that calling relevant bits of code adds the right paper to citations
@@ -235,6 +268,7 @@ class TestCitations(unittest.TestCase):
             build=False, options={"lithium plating": "irreversible"}
         )
         self.assertIn("BrosaPlanella2022", citations._papers_to_cite)
+        self.assertIn("BrosaPlanella2022", citations._citation_tags.keys())
         citations._reset()
 
     def test_newman_tobias(self):
@@ -246,7 +280,9 @@ class TestCitations(unittest.TestCase):
         self.assertNotIn("Chu2020", citations._papers_to_cite)
         pybamm.lithium_ion.NewmanTobias()
         self.assertIn("Newman1962", citations._papers_to_cite)
+        self.assertIn("Newman1962", citations._citation_tags.keys())
         self.assertIn("Chu2020", citations._papers_to_cite)
+        self.assertIn("Chu2020", citations._citation_tags.keys())
 
     def test_scikit_fem(self):
         citations = pybamm.citations
@@ -255,6 +291,7 @@ class TestCitations(unittest.TestCase):
         self.assertNotIn("Gustafsson2020", citations._papers_to_cite)
         pybamm.ScikitFiniteElement()
         self.assertIn("Gustafsson2020", citations._papers_to_cite)
+        self.assertIn("Gustafsson2020", citations._citation_tags.keys())
 
     def test_reniers_2019(self):
         # Test that calling relevant bits of code adds the right paper to citations
@@ -264,6 +301,7 @@ class TestCitations(unittest.TestCase):
         self.assertNotIn("Reniers2019", citations._papers_to_cite)
         pybamm.active_material.LossActiveMaterial(None, "negative", None, True)
         self.assertIn("Reniers2019", citations._papers_to_cite)
+        self.assertIn("Reniers2019", citations._citation_tags.keys())
 
     def test_mohtat_2019(self):
         citations = pybamm.citations
@@ -274,6 +312,7 @@ class TestCitations(unittest.TestCase):
             pybamm.ParameterValues("Marquis2019")
         )._get_electrode_soh_sims_full()
         self.assertIn("Mohtat2019", citations._papers_to_cite)
+        self.assertIn("Mohtat2019", citations._citation_tags.keys())
 
     def test_mohtat_2021(self):
         citations = pybamm.citations
@@ -282,6 +321,7 @@ class TestCitations(unittest.TestCase):
         self.assertNotIn("Mohtat2021", citations._papers_to_cite)
         pybamm.external_circuit.CCCVFunctionControl(None, None)
         self.assertIn("Mohtat2021", citations._papers_to_cite)
+        self.assertIn("Mohtat2021", citations._citation_tags.keys())
 
     def test_sripad_2020(self):
         citations = pybamm.citations
@@ -290,11 +330,13 @@ class TestCitations(unittest.TestCase):
         self.assertNotIn("Sripad2020", citations._papers_to_cite)
         pybamm.kinetics.Marcus(None, None, None, None, None)
         self.assertIn("Sripad2020", citations._papers_to_cite)
+        self.assertIn("Sripad2020", citations._citation_tags.keys())
 
         citations._reset()
         self.assertNotIn("Sripad2020", citations._papers_to_cite)
         pybamm.kinetics.MarcusHushChidsey(None, None, None, None, None)
         self.assertIn("Sripad2020", citations._papers_to_cite)
+        self.assertIn("Sripad2020", citations._citation_tags.keys())
 
     def test_parameter_citations(self):
         citations = pybamm.citations
@@ -302,30 +344,40 @@ class TestCitations(unittest.TestCase):
         citations._reset()
         pybamm.ParameterValues("Chen2020")
         self.assertIn("Chen2020", citations._papers_to_cite)
+        self.assertIn("Chen2020", citations._citation_tags.keys())
 
         citations._reset()
         pybamm.ParameterValues("NCA_Kim2011")
         self.assertIn("Kim2011", citations._papers_to_cite)
+        self.assertIn("Kim2011", citations._citation_tags.keys())
 
         citations._reset()
         pybamm.ParameterValues("Marquis2019")
         self.assertIn("Marquis2019", citations._papers_to_cite)
+        self.assertIn("Marquis2019", citations._citation_tags.keys())
 
         citations._reset()
         pybamm.ParameterValues("Sulzer2019")
         self.assertIn("Sulzer2019physical", citations._papers_to_cite)
+        self.assertIn("Sulzer2019physical", citations._citation_tags.keys())
 
         citations._reset()
         pybamm.ParameterValues("Ecker2015")
         self.assertIn("Ecker2015i", citations._papers_to_cite)
+        self.assertIn("Ecker2015i", citations._citation_tags.keys())
         self.assertIn("Ecker2015ii", citations._papers_to_cite)
+        self.assertIn("Ecker2015ii", citations._citation_tags.keys())
         self.assertIn("Zhao2018", citations._papers_to_cite)
+        self.assertIn("Zhao2018", citations._citation_tags.keys())
         self.assertIn("Hales2019", citations._papers_to_cite)
+        self.assertIn("Hales2019", citations._citation_tags.keys())
         self.assertIn("Richardson2020", citations._papers_to_cite)
+        self.assertIn("Richardson2020", citations._citation_tags.keys())
 
         citations._reset()
         pybamm.ParameterValues("ORegan2022")
         self.assertIn("ORegan2022", citations._papers_to_cite)
+        self.assertIn("ORegan2022", citations._citation_tags.keys())
 
     def test_solver_citations(self):
         # Test that solving each solver adds the right citations
@@ -335,28 +387,33 @@ class TestCitations(unittest.TestCase):
         self.assertNotIn("Virtanen2020", citations._papers_to_cite)
         pybamm.ScipySolver()
         self.assertIn("Virtanen2020", citations._papers_to_cite)
+        self.assertIn("Virtanen2020", citations._citation_tags.keys())
 
         citations._reset()
         self.assertNotIn("Virtanen2020", citations._papers_to_cite)
         pybamm.AlgebraicSolver()
         self.assertIn("Virtanen2020", citations._papers_to_cite)
+        self.assertIn("Virtanen2020", citations._citation_tags.keys())
 
         if pybamm.have_scikits_odes():
             citations._reset()
             self.assertNotIn("Malengier2018", citations._papers_to_cite)
             pybamm.ScikitsOdeSolver()
             self.assertIn("Malengier2018", citations._papers_to_cite)
+            self.assertIn("Malengier2018", citations._citation_tags.keys())
 
             citations._reset()
             self.assertNotIn("Malengier2018", citations._papers_to_cite)
             pybamm.ScikitsDaeSolver()
             self.assertIn("Malengier2018", citations._papers_to_cite)
+            self.assertIn("Malengier2018", citations._citation_tags.keys())
 
         if pybamm.have_idaklu():
             citations._reset()
             self.assertNotIn("Hindmarsh2005", citations._papers_to_cite)
             pybamm.IDAKLUSolver()
             self.assertIn("Hindmarsh2005", citations._papers_to_cite)
+            self.assertIn("Hindmarsh2005", citations._citation_tags.keys())
 
     @unittest.skipIf(not pybamm.have_jax(), "jax or jaxlib is not installed")
     def test_jax_citations(self):
@@ -365,6 +422,7 @@ class TestCitations(unittest.TestCase):
         self.assertNotIn("jax2018", citations._papers_to_cite)
         pybamm.JaxSolver()
         self.assertIn("jax2018", citations._papers_to_cite)
+        self.assertIn("jax2018", citations._citation_tags.keys())
 
 
 if __name__ == "__main__":