Repository: theochem/AtomDB (language: python)

Issue:
Improve periodic module (generate class)
Improves the periodic.py module by reading the CSV data files _only once_ and then generating a class with the proper properties.



Discussion / hints:
@gabrielasd @marco-2023 I did the following:

- Improve species class (DRY)

- Clean up module structure

- Split datasets' docstrings and run functions

This will require re-compiling datasets (which we already require by making species fields more consistent, but should be future-proof from there.

Please let me know what you think.

Fix patch:
diff --git a/atomdb/api.py b/atomdb/api.py
index ae46a169..d43f45b2 100644
--- a/atomdb/api.py
+++ b/atomdb/api.py
@@ -769,5 +769,8 @@ def get_element_data(elem):
 
 def get_docstring(dataset):
     """Retrieve the dataset doctring."""
-    submodule = import_module(f"atomdb.datasets.{dataset}")
-    return submodule.DOCSTRING
+    # FIXME: This function breacks the inicialization of the Species class.
+    # The databases docstrings need to be separated from the compilation scripts files.
+    # submodule = import_module(f"atomdb.datasets.{dataset}")
+    # return submodule.DOCSTRING
+    return "No documentation available."
diff --git a/pyproject.toml b/pyproject.toml
index 8872601b..c8f9ee62 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -71,6 +71,17 @@ doc = [
     "docutils==0.16",  # Needed to show bullet points in sphinx_rtd_theme
     "nbsphinx-link"
 ]
+dev = [
+    "pyscf",
+    "qc-gbasis@git+https://github.com/theochem/gbasis.git@master",
+    "qc-grid@git+https://github.com/theochem/grid.git@master",
+    "qc-iodata@git+https://github.com/theochem/iodata.git@master"
+]
+test_extra = [
+    "pytest-md",
+    "pytest-emoji",
+    "pytest-cov",
+]
 
 [tool.black]
 line-length = 100
@@ -91,8 +102,10 @@ extend-exclude = ["doc/*", "doc/*/*"]
 
 [tool.pytest.ini_options]
 minversion = "7.0"
-testpaths = ["tests"]
-addopts = "-m 'not dev'"
+testpaths = ["test"]
+addopts = "-m 'not dev' --import-mode=importlib"
 markers = [
     "dev: mark a developer test, needs extra dependencies.",
 ]
+
+


Test patch:
diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
index 84957f09..fd361c68 100644
--- a/.github/workflows/pytest.yaml
+++ b/.github/workflows/pytest.yaml
@@ -26,18 +26,14 @@ jobs:
         with:
           python-version: ${{ matrix.py }}
 
-      - name: Install dependencies for testing
-        run: |
-          pip install --upgrade pip
-          pip install pytest pytest-md pytest-emoji pytest-cov pyscf
-          pip install git+https://github.com/theochem/gbasis.git@master
-          pip install git+https://github.com/theochem/grid.git@master
-          pip install git+https://github.com/theochem/iodata.git@master
-
       - name: Install development version
         run: |
           pip install -v .
 
+      - name: Install extra test dependencies
+        run: |
+          pip install .[test_extra]
+
       - name: Run pytest default tests
         uses: pavelzw/pytest-action@v2
         with:
@@ -47,6 +43,10 @@ jobs:
           click-to-expand: true
           report-title: 'Test Report'
 
+      - name: Install Dev Dependencies
+        run: |
+          pip install .[dev]
+
       - name: Run pytest Dev Tests
         uses: pavelzw/pytest-action@v2
         with:
diff --git a/atomdb/test/test_gaussian.py b/atomdb/test/test_gaussian.py
index 602885fb..36367eea 100644
--- a/atomdb/test/test_gaussian.py
+++ b/atomdb/test/test_gaussian.py
@@ -27,7 +27,7 @@
 from atomdb.api import load
 from importlib_resources import files
 import numpy as np
-from numpy.testing import assert_equal, assert_almost_equal
+from numpy.testing import assert_almost_equal
 import os
 
 # get test data path
@@ -35,167 +35,6 @@
 TEST_DATAPATH = os.fspath(TEST_DATAPATH._paths[0])
 
 
-@pytest.mark.dev
-@pytest.mark.parametrize(
-    "atom", ["atom_018_Ar_N18_M1_uhf_def2svpd_g09.fchk", "atom_001_H_N02_M1_uhf_def2svpd_g09.fchk"]
-)
-def test_eval_radial_d_density(atom):
-    # conditional import of dev modules needed for this test
-    try:
-        from atomdb.datasets.gaussian import eval_radial_d_density
-        from gbasis.wrappers import from_iodata
-        from gbasis.evals.density import evaluate_density_gradient
-        from grid import UniformInteger, LinearInfiniteRTransform, AtomGrid
-        from iodata import load_one
-
-        assert True
-    except ImportError:
-        pytest.skip("Gbasis or IOData modules not available, skipping test")
-
-    # create atomic grid from 0 to 20 bohr
-    oned = UniformInteger(npoints=100)
-    rgrid = LinearInfiniteRTransform(1e-4, 20).transform_1d_grid(oned)
-    atgrid = AtomGrid(rgrid, degrees=[10])
-
-    # load the fchk file
-    mol_data = load_one(os.path.join(TEST_DATAPATH, "gaussian", atom))
-    ao_basis = from_iodata(mol_data)
-
-    # one electron RDM from fchk file
-    rdm = mol_data.one_rdms["scf"]
-
-    # evaluate gradient of the electron density on the grid
-    rho_grad = evaluate_density_gradient(rdm, ao_basis, atgrid.points)
-    # evaluate derivative of rho vs r on the grid
-    radial_d_rho = eval_radial_d_density(rdm, ao_basis, atgrid.points)
-    # compute unitary projection of r in x,y,z
-    unitvects = atgrid.points / np.linalg.norm(atgrid.points, axis=1)[:, None]
-
-    # recover cartesian gradient from radial derivative (for spheric atoms) and compare
-    rho_grad_cart = radial_d_rho[:, None] * unitvects
-
-    assert np.allclose(rho_grad, rho_grad_cart, rtol=1e-6)
-
-
-@pytest.mark.dev
-@pytest.mark.parametrize(
-    "atom", ["atom_018_Ar_N18_M1_uhf_def2svpd_g09.fchk", "atom_001_H_N02_M1_uhf_def2svpd_g09.fchk"]
-)
-def test_eval_radial_dd_density(atom):
-    # conditional import of dev modules needed for this test
-    try:
-        from atomdb.datasets.gaussian import eval_radial_d_density, eval_radial_dd_density
-        from gbasis.wrappers import from_iodata
-        from gbasis.evals.density import evaluate_density_laplacian
-        from grid import UniformInteger, LinearInfiniteRTransform, AtomGrid
-        from iodata import load_one
-
-        assert True
-    except ImportError:
-        pytest.skip("Gbasis or IOData modules not available, skipping test")
-
-    # create atomic grid from 0 to 20 bohr
-    oned = UniformInteger(npoints=100)
-    rgrid = LinearInfiniteRTransform(1e-4, 20).transform_1d_grid(oned)
-    atgrid = AtomGrid(rgrid, degrees=[10])
-
-    # load the fchk file
-    mol_data = load_one(os.path.join(TEST_DATAPATH, "gaussian", atom))
-    ao_basis = from_iodata(mol_data)
-
-    # one electron RDM from fchk file
-    rdm = mol_data.one_rdms["scf"]
-
-    # evaluate fisrt and second derivatives of rho vs r on the grid
-    radial_d_rho = eval_radial_d_density(rdm, ao_basis, atgrid.points)
-    radial_dd_rho = eval_radial_dd_density(rdm, ao_basis, atgrid.points)
-
-    # evaluate gradient, hessian and laplacian of the electron density on the grid
-    rho_lapl = evaluate_density_laplacian(rdm, ao_basis, atgrid.points)
-
-    # compute laplacian from first and second radial derivatives (for spheric atoms) and compare
-    rho_lapl_rec = radial_dd_rho + 2 * radial_d_rho / np.linalg.norm(atgrid.points, axis=1)
-    assert np.allclose(rho_lapl, rho_lapl_rec, rtol=1e-6)
-
-
-@pytest.mark.dev
-@pytest.mark.parametrize(
-    "atom", ["atom_018_Ar_N18_M1_uhf_def2svpd_g09.fchk", "atom_001_H_N02_M1_uhf_def2svpd_g09.fchk"]
-)
-def test_eval_orbs_radial_d_density(atom):
-    # conditional import of dev modules needed for this test
-    try:
-
-        from atomdb.datasets.gaussian import eval_radial_d_density, eval_orbs_radial_d_density
-        from gbasis.wrappers import from_iodata
-        from grid import UniformInteger, LinearInfiniteRTransform, AtomGrid
-        from iodata import load_one
-
-        assert True
-    except ImportError:
-        pytest.skip("Gbasis or IOData modules not available, skipping test")
-
-    # create atomic grid from 0 to 20 bohr
-    oned = UniformInteger(npoints=100)
-    rgrid = LinearInfiniteRTransform(1e-4, 20).transform_1d_grid(oned)
-    atgrid = AtomGrid(rgrid, degrees=[10])
-
-    # load the fchk file
-    mol_data = load_one(os.path.join(TEST_DATAPATH, "gaussian", atom))
-    ao_basis = from_iodata(mol_data)
-
-    # one electron RDM and MO coefficients from fchk file
-    rdm = mol_data.one_rdms["scf"]
-
-    # evaluate derivative of rho vs r on the grid
-    radial_d_rho = eval_radial_d_density(rdm, ao_basis, atgrid.points)
-
-    # evaluate radial derivative of the orbital densities on the grid
-    radial_d_rho_orbs = eval_orbs_radial_d_density(rdm, ao_basis, atgrid.points)
-    # compute total density from orbital densities
-    radial_d_rho_from_orbs = np.einsum("ij->j", radial_d_rho_orbs)
-
-    assert np.allclose(radial_d_rho_from_orbs, radial_d_rho, rtol=1e-6)
-
-
-@pytest.mark.dev
-@pytest.mark.parametrize(
-    "atom", ["atom_018_Ar_N18_M1_uhf_def2svpd_g09.fchk", "atom_001_H_N02_M1_uhf_def2svpd_g09.fchk"]
-)
-def test_eval_orbs_radial_dd_density(atom):
-    # conditional import of dev modules needed for this test
-    try:
-        from atomdb.datasets.gaussian import eval_radial_dd_density, eval_orbs_radial_dd_density
-        from gbasis.wrappers import from_iodata
-        from grid import UniformInteger, LinearInfiniteRTransform, AtomGrid
-        from iodata import load_one
-
-        assert True
-    except ImportError:
-        pytest.skip("Gbasis or IOData modules not available, skipping test")
-
-    # create atomic grid from 0 to 20 bohr
-    oned = UniformInteger(npoints=100)
-    rgrid = LinearInfiniteRTransform(1e-4, 20).transform_1d_grid(oned)
-    atgrid = AtomGrid(rgrid, degrees=[10])
-
-    # load the fchk file
-    mol_data = load_one(os.path.join(TEST_DATAPATH, "gaussian", atom))
-    ao_basis = from_iodata(mol_data)
-
-    # one electron RDM from fchk file
-    rdm = mol_data.one_rdms["scf"]
-
-    # evaluate second derivatives of rho vs r on the grid
-    radial_dd_rho = eval_radial_dd_density(rdm, ao_basis, atgrid.points)
-    # evaluate second derivatives of the orbital densities on the grid
-    radial_dd_rho_orbs = eval_orbs_radial_dd_density(rdm, ao_basis, atgrid.points)
-    # compute total second derivative of the density
-    recov_radial_dd_rho = np.sum(radial_dd_rho_orbs, axis=0)
-    # compare with the total second derivative of the density
-    assert np.allclose(recov_radial_dd_rho, radial_dd_rho, rtol=1e-6)
-
-
 def test_compiled_gaussian_hf_data():
     ### Use Be atomic data as a test case
     sp = load("Be", 0, 1, dataset="gaussian", datapath=TEST_DATAPATH)
diff --git a/atomdb/test/test_gaussian_dev.py b/atomdb/test/test_gaussian_dev.py
new file mode 100644
index 00000000..c34fecff
--- /dev/null
+++ b/atomdb/test/test_gaussian_dev.py
@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+# AtomDB is an extended periodic table database containing experimental
+# and/or computational information on stable ground state
+# and/or excited states of neutral and charged atomic species.
+#
+# Copyright (C) 2014-2015 The AtomDB Development Team
+#
+# This file is part of AtomDB.
+#
+# AtomDB is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# AtomDB is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>
+#
+# --
+
+import pytest
+import numpy as np
+import os
+from importlib_resources import files
+
+# avoid import error on 'not dev' tests, these statements are run even when the tests are skipped
+try:
+    from atomdb.datasets.gaussian import eval_radial_d_density, eval_radial_dd_density
+    from atomdb.datasets.gaussian import eval_orbs_radial_d_density, eval_orbs_radial_dd_density
+    from gbasis.wrappers import from_iodata
+    from gbasis.evals.density import evaluate_density_gradient, evaluate_density_laplacian
+    from grid import UniformInteger, LinearInfiniteRTransform, AtomGrid
+    from iodata import load_one
+except ImportError:
+    pass
+
+# mark all tests in this file as development tests
+pytestmark = pytest.mark.dev
+
+# get test data path
+TEST_DATAPATH = files("atomdb.test.data")
+TEST_DATAPATH = os.fspath(TEST_DATAPATH._paths[0])
+
+
+@pytest.mark.parametrize(
+    "atom", ["atom_018_Ar_N18_M1_uhf_def2svpd_g09.fchk", "atom_001_H_N02_M1_uhf_def2svpd_g09.fchk"]
+)
+def test_eval_radial_d_density(atom):
+    # create atomic grid from 0 to 20 bohr
+    oned = UniformInteger(npoints=100)
+    rgrid = LinearInfiniteRTransform(1e-4, 20).transform_1d_grid(oned)
+    atgrid = AtomGrid(rgrid, degrees=[10])
+
+    # load the fchk file
+    mol_data = load_one(os.path.join(TEST_DATAPATH, "gaussian", atom))
+    ao_basis = from_iodata(mol_data)
+
+    # one electron RDM from fchk file
+    rdm = mol_data.one_rdms["scf"]
+
+    # evaluate gradient of the electron density on the grid
+    rho_grad = evaluate_density_gradient(rdm, ao_basis, atgrid.points)
+    # evaluate derivative of rho vs r on the grid
+    radial_d_rho = eval_radial_d_density(rdm, ao_basis, atgrid.points)
+    # compute unitary projection of r in x,y,z
+    unitvects = atgrid.points / np.linalg.norm(atgrid.points, axis=1)[:, None]
+
+    # recover cartesian gradient from radial derivative (for spheric atoms) and compare
+    rho_grad_cart = radial_d_rho[:, None] * unitvects
+
+    assert np.allclose(rho_grad, rho_grad_cart, rtol=1e-6)
+
+
+@pytest.mark.parametrize(
+    "atom", ["atom_018_Ar_N18_M1_uhf_def2svpd_g09.fchk", "atom_001_H_N02_M1_uhf_def2svpd_g09.fchk"]
+)
+def test_eval_radial_dd_density(atom):
+    # create atomic grid from 0 to 20 bohr
+    oned = UniformInteger(npoints=100)
+    rgrid = LinearInfiniteRTransform(1e-4, 20).transform_1d_grid(oned)
+    atgrid = AtomGrid(rgrid, degrees=[10])
+
+    # load the fchk file
+    mol_data = load_one(os.path.join(TEST_DATAPATH, "gaussian", atom))
+    ao_basis = from_iodata(mol_data)
+
+    # one electron RDM from fchk file
+    rdm = mol_data.one_rdms["scf"]
+
+    # evaluate fisrt and second derivatives of rho vs r on the grid
+    radial_d_rho = eval_radial_d_density(rdm, ao_basis, atgrid.points)
+    radial_dd_rho = eval_radial_dd_density(rdm, ao_basis, atgrid.points)
+
+    # evaluate gradient, hessian and laplacian of the electron density on the grid
+    rho_lapl = evaluate_density_laplacian(rdm, ao_basis, atgrid.points)
+
+    # compute laplacian from first and second radial derivatives (for spheric atoms) and compare
+    rho_lapl_rec = radial_dd_rho + 2 * radial_d_rho / np.linalg.norm(atgrid.points, axis=1)
+    assert np.allclose(rho_lapl, rho_lapl_rec, rtol=1e-6)
+
+
+@pytest.mark.dev
+@pytest.mark.parametrize(
+    "atom", ["atom_018_Ar_N18_M1_uhf_def2svpd_g09.fchk", "atom_001_H_N02_M1_uhf_def2svpd_g09.fchk"]
+)
+def test_eval_orbs_radial_d_density(atom):
+    # create atomic grid from 0 to 20 bohr
+    oned = UniformInteger(npoints=100)
+    rgrid = LinearInfiniteRTransform(1e-4, 20).transform_1d_grid(oned)
+    atgrid = AtomGrid(rgrid, degrees=[10])
+
+    # load the fchk file
+    mol_data = load_one(os.path.join(TEST_DATAPATH, "gaussian", atom))
+    ao_basis = from_iodata(mol_data)
+
+    # one electron RDM and MO coefficients from fchk file
+    rdm = mol_data.one_rdms["scf"]
+
+    # evaluate derivative of rho vs r on the grid
+    radial_d_rho = eval_radial_d_density(rdm, ao_basis, atgrid.points)
+
+    # evaluate radial derivative of the orbital densities on the grid
+    radial_d_rho_orbs = eval_orbs_radial_d_density(rdm, ao_basis, atgrid.points)
+    # compute total density from orbital densities
+    radial_d_rho_from_orbs = np.einsum("ij->j", radial_d_rho_orbs)
+
+    assert np.allclose(radial_d_rho_from_orbs, radial_d_rho, rtol=1e-6)
+
+
+@pytest.mark.dev
+@pytest.mark.parametrize(
+    "atom", ["atom_018_Ar_N18_M1_uhf_def2svpd_g09.fchk", "atom_001_H_N02_M1_uhf_def2svpd_g09.fchk"]
+)
+def test_eval_orbs_radial_dd_density(atom):
+    # create atomic grid from 0 to 20 bohr
+    oned = UniformInteger(npoints=100)
+    rgrid = LinearInfiniteRTransform(1e-4, 20).transform_1d_grid(oned)
+    atgrid = AtomGrid(rgrid, degrees=[10])
+
+    # load the fchk file
+    mol_data = load_one(os.path.join(TEST_DATAPATH, "gaussian", atom))
+    ao_basis = from_iodata(mol_data)
+
+    # one electron RDM from fchk file
+    rdm = mol_data.one_rdms["scf"]
+
+    # evaluate second derivatives of rho vs r on the grid
+    radial_dd_rho = eval_radial_dd_density(rdm, ao_basis, atgrid.points)
+    # evaluate second derivatives of the orbital densities on the grid
+    radial_dd_rho_orbs = eval_orbs_radial_dd_density(rdm, ao_basis, atgrid.points)
+    # compute total second derivative of the density
+    recov_radial_dd_rho = np.sum(radial_dd_rho_orbs, axis=0)
+    # compare with the total second derivative of the density
+    assert np.allclose(recov_radial_dd_rho, radial_dd_rho, rtol=1e-6)