Repository: pyscf/pyscf — pull request #1841 (python)
Title: Rewrite _break_dm_spin_symm

Rewrite _break_dm_spin_symm to form a doublet state density instead of zeroing out interatomic blocks of density matrix.

Closes #1839 

Resolved issues:
title: strange behavior of init_guess_breaksym
body: Hi, 

I'm wondering if the breaking symmetry option 

https://github.com/pyscf/pyscf/blob/master/pyscf/scf/uhf.py#L29
https://github.com/pyscf/pyscf/blob/master/pyscf/scf/uhf.py#L773

works as it is supposed to do. 
I would propose not to turn on the option by default 
for the initial density matrix methods. 
For augmented basis sets, e.g., aug-pc-n (n=1,2..), 
the number of electrons in the beta channel explodes 
using MINAO with `init_guess_breaksym=True`. 
The problem is reproducible with all 2.X.Y PySCF versions. 
For benzene, this leads already to significant convergence issues. 

Please see the attached script to check to problem yourself. 


**Tab. 1 initial_guess_dm=True, PySCF version: 2.3.0**
| basis| N$`_{elec}`$(from initial guess) = (N$`_{\alpha}`$, N$`_{\beta}`$)  |
| :--: | :-----------------: |
| pc-0 | (20.951768099569655, 20.91592508353395) |
| pc-1 | (20.992586040795963, 22.024225871340562) | 
| pc-2 | (20.99925998574021, 20.614775142071956) | 
| aug-pc-0 | (20.964939403069046, 24.754095109846062) | 
| aug-pc-1 | (20.996679459866204, 39.278482611398196) | 
| aug-pc-2 | (20.99947479330699, 352.8876628321081)| 


**Tab. 2 initial_guess_dm=False, PySCF version: 2.3.0**
| basis| N$`_{elec}`$(from initial guess) = (N$`_{\alpha}`$, N$`_{\beta}`$)  |
| :--: | :-----------------: |
| pc-0 | (20.951768099569655, 20.951768099569655)| 
| pc-1 | (20.992586040795963, 20.992586040795963) | 
| pc-2 | (20.99925998574021, 20.99925998574021) |
| aug-pc-0 | (20.964939403069046, 20.964939403069046)| 
| aug-pc-1 | (20.996679459866204, 20.996679459866204)| 
| aug-pc-2 | (20.99947479330699, 20.99947479330699) | 

```python=
import pyscf
from pyscf import gto, scf


def dft(basis="pc-0",init_guess_breaksym=True):
    mol = gto.M(atom='''
C       0.000000000000000      1.391976000000000      0.000000000000000
C       1.205486000000000      0.695988000000000      0.000000000000000
C       1.205486000000000     -0.695988000000000      0.000000000000000
C       0.000000000000000     -1.391976000000000      0.000000000000000
C      -1.205486000000000     -0.695988000000000      0.000000000000000
C      -1.205486000000000      0.695988000000000      0.000000000000000
H       0.000000000000000      2.472230000000000      0.000000000000000
H       2.141014000000000      1.236115000000000      0.000000000000000
H       2.141014000000000     -1.236115000000000      0.000000000000000
H       0.000000000000000     -2.472230000000000      0.000000000000000
H      -2.141014000000000     -1.236115000000000      0.000000000000000
H      -2.141014000000000      1.236115000000000      0.000000000000000
''')

    mol.verbose = 9
    mol.basis = basis
    mol.output = "dft.log"
    mol.build()
    mf = scf.UKS(mol)
    mf.init_guess_breaksym = init_guess_breaksym
    mf.max_cycle = 0
    mf.xc = "spw92"
    mf.conv_tol = 1e-8
    mf.grids.level = 7
    mf.grids.prune = None
    mf.grids.build()
    # mf.init_guess = "vsap"
    edft = mf.kernel()

def main():
    print(f"PySCF version: {pyscf.__version__}")
    dft(basis="aug-pc-1",init_guess_breaksym=True)

if __name__ == '__main__':
    main()

```

Best, 
Wanja, Kai, and Sebastian 


### Minimal API referenced by tests (typed signatures)
_Added only when tests call brand‑new symbols introduced by the fix; types reflect annotations present in the repo or `Unknown` if not annotated._

- Function in `pyscf/scf/uhf.py`: `def init_guess_by_minao(mol, breaksym) -> numpy.ndarray: ...`
  **Inputs:** mol: gto.Mole, breaksym: bool
  **Output:** numpy.ndarray

- Function in `pyscf/scf/uhf.py`: `def init_guess_by_1e(mol, breaksym) -> numpy.ndarray: ...`
  **Inputs:** mol: gto.Mole, breaksym: bool
  **Output:** numpy.ndarray

```python
def init_guess_by_minao(mol, breaksym) -> numpy.ndarray: ...
def init_guess_by_1e(mol, breaksym) -> numpy.ndarray: ...
```

Hints:

Fix patch:
diff --git a/pyscf/pbc/scf/kuhf.py b/pyscf/pbc/scf/kuhf.py
index ff911afac4..8827cbad6e 100644
--- a/pyscf/pbc/scf/kuhf.py
+++ b/pyscf/pbc/scf/kuhf.py
@@ -369,6 +369,7 @@ class KUHF(khf.KSCF):
     '''UHF class with k-point sampling.
     '''
     conv_tol_grad = getattr(__config__, 'pbc_scf_KSCF_conv_tol_grad', None)
+    init_guess_breaksym = getattr(__config__, 'scf_uhf_init_guess_breaksym', 1)
 
     _keys = {"init_guess_breaksym"}
 
@@ -390,7 +391,6 @@ def __init__(self, cell, kpts=np.zeros((1,3)),
                  exxdiv=getattr(__config__, 'pbc_scf_SCF_exxdiv', 'ewald')):
         khf.KSCF.__init__(self, cell, kpts, exxdiv)
         self.nelec = None
-        self.init_guess_breaksym = None
 
     @property
     def nelec(self):
diff --git a/pyscf/pbc/scf/uhf.py b/pyscf/pbc/scf/uhf.py
index 3dfd53866b..ea48c75efb 100644
--- a/pyscf/pbc/scf/uhf.py
+++ b/pyscf/pbc/scf/uhf.py
@@ -107,6 +107,8 @@ def dip_moment(cell, dm, unit='Debye', verbose=logger.NOTE,
 class UHF(pbchf.SCF):
     '''UHF class for PBCs.
     '''
+    init_guess_breaksym = getattr(__config__, 'scf_uhf_init_guess_breaksym', 1)
+
     _keys = {"init_guess_breaksym"}
 
     init_guess_by_minao  = mol_uhf.UHF.init_guess_by_minao
@@ -134,7 +136,6 @@ def __init__(self, cell, kpt=np.zeros(3),
                  exxdiv=getattr(__config__, 'pbc_scf_SCF_exxdiv', 'ewald')):
         pbchf.SCF.__init__(self, cell, kpt, exxdiv)
         self.nelec = None
-        self.init_guess_breaksym = None
 
     @property
     def nelec(self):
diff --git a/pyscf/scf/uhf.py b/pyscf/scf/uhf.py
index 4797dbe5d9..c4c1660ceb 100644
--- a/pyscf/scf/uhf.py
+++ b/pyscf/scf/uhf.py
@@ -26,48 +26,34 @@
 
 WITH_META_LOWDIN = getattr(__config__, 'scf_analyze_with_meta_lowdin', True)
 PRE_ORTH_METHOD = getattr(__config__, 'scf_analyze_pre_orth_method', 'ANO')
-BREAKSYM = getattr(__config__, 'scf_uhf_init_guess_breaksym', True)
 MO_BASE = getattr(__config__, 'MO_BASE', 1)
 
 
-def init_guess_by_minao(mol, breaksym=BREAKSYM):
+def init_guess_by_minao(mol, breaksym=None):
     '''Generate initial guess density matrix based on ANO basis, then project
     the density matrix to the basis set defined by ``mol``
 
     Returns:
         Density matrices, a list of 2D ndarrays for alpha and beta spins
     '''
-    dm = hf.init_guess_by_minao(mol)
-    dma = dmb = dm*.5
-    if breaksym:
-        dma, dmb = _break_dm_spin_symm(mol, (dma, dmb))
-    return numpy.array((dma,dmb))
+    return UHF(mol).init_guess_by_minao(mol, breaksym)
 
-def init_guess_by_1e(mol, breaksym=BREAKSYM):
+def init_guess_by_1e(mol, breaksym=None):
     return UHF(mol).init_guess_by_1e(mol, breaksym)
 
-def init_guess_by_atom(mol, breaksym=BREAKSYM):
-    dm = hf.init_guess_by_atom(mol)
-    dma = dmb = dm*.5
-    if mol.spin == 0 and breaksym:
-        #Add off-diagonal part for alpha DM
-        dma = mol.intor_symmetric('int1e_ovlp') * 1e-2
-        for b0, b1, p0, p1 in mol.aoslice_by_atom():
-            dma[p0:p1,p0:p1] = dmb[p0:p1,p0:p1]
-    return numpy.array((dma,dmb))
-
-def init_guess_by_huckel(mol, breaksym=BREAKSYM):
+def init_guess_by_atom(mol, breaksym=None):
+    return UHF(mol).init_guess_by_atom(mol, breaksym)
+
+def init_guess_by_huckel(mol, breaksym=None):
     return UHF(mol).init_guess_by_huckel(mol, breaksym)
 
-def init_guess_by_mod_huckel(mol, breaksym=BREAKSYM):
+def init_guess_by_mod_huckel(mol, breaksym=None):
     return UHF(mol).init_guess_by_mod_huckel(mol, breaksym)
 
-def init_guess_by_sap(mol, sap_basis, breaksym=BREAKSYM, **kwargs):
-    dm = hf.init_guess_by_sap(mol, sap_basis)
-    dma = dmb = dm*.5
-    if breaksym:
-        dma, dmb = _break_dm_spin_symm(mol, (dma, dmb))
-    return numpy.array((dma,dmb))
+def init_guess_by_sap(mol, sap_basis, breaksym=None, **kwargs):
+    mf = UHF(mol)
+    mf.sap_basis = sap_basis
+    return mf.init_guess_by_sap(mol, breaksym)
 
 def init_guess_by_chkfile(mol, chkfile_name, project=None):
     '''Read SCF chkfile and make the density matrix for UHF initial guess.
@@ -127,14 +113,24 @@ def fproj(mo):
         dm = make_rdm1([fproj(mo[0]),fproj(mo[1])], mo_occ)
     return dm
 
-def _break_dm_spin_symm(mol, dm):
+def _break_dm_spin_symm(mol, dm, breaksym=1):
     dma, dmb = dm
     # For spin polarized system, no need to manually break spin symmetry
-    if mol.spin == 0 and abs(dma - dmb).max() < 1e-2:
-        #remove off-diagonal part of beta DM
-        dmb = numpy.zeros_like(dma)
-        for b0, b1, p0, p1 in mol.aoslice_by_atom():
-            dmb[...,p0:p1,p0:p1] = dma[...,p0:p1,p0:p1]
+    if breaksym and mol.spin == 0 and abs(dma - dmb).max() < 1e-2:
+        if breaksym == 1:
+            #remove off-diagonal part of beta DM
+            dmb = numpy.zeros_like(dma)
+            for b0, b1, p0, p1 in mol.aoslice_by_atom():
+                dmb[...,p0:p1,p0:p1] = dma[...,p0:p1,p0:p1]
+        else:
+            # Adjust num. electrons for density matrices (issue #1839)
+            # Get overlap matrix
+            s1e = mol.intor_symmetric('int1e_ovlp')
+            # Compute norm of density matrices
+            nelec_half = numpy.einsum('ij,ji->', dma, s1e)
+            # Scale density matrices to form doublet state
+            dma = dma * (nelec_half+1) / nelec_half
+            dmb = dmb * (nelec_half-1) / nelec_half
     return dma, dmb
 
 def get_init_guess(mol, key='minao', **kwargs):
@@ -757,8 +753,12 @@ class UHF(hf.SCF):
             If given, freeze the number of (alpha,beta) electrons to the given value.
         level_shift : number or two-element list
             level shift (in Eh) for alpha and beta Fock if two-element list is given.
-        init_guess_breaksym : logical
-            If given, overwrite BREAKSYM.
+        init_guess_breaksym : int
+             This configuration controls the algorithm used to break the spin
+             symmetry of the initial guess:
+             - 0 to disable symmetry breaking in the initial guess.
+             - 1 to use the default algorithm introduced in pyscf-1.7.
+             - 2 to adjust the num. electrons for spin-up and spin-down density matrices (issue #1839).
 
     Examples:
 
@@ -770,7 +770,7 @@ class UHF(hf.SCF):
     S^2 = 0.7570150, 2S+1 = 2.0070027
     '''
 
-    init_guess_breaksym = None
+    init_guess_breaksym = getattr(__config__, 'scf_uhf_init_guess_breaksym', 1)
 
     _keys = {"init_guess_breaksym"}
 
@@ -847,29 +847,34 @@ def get_init_guess(self, mol=None, key='minao', **kwargs):
             logger.debug1(self, 'Nelec from initial guess = %s', nelec)
         return dm
 
-    def init_guess_by_minao(self, mol=None, breaksym=BREAKSYM):
+    def init_guess_by_minao(self, mol=None, breaksym=None):
         '''Initial guess in terms of the overlap to minimal basis.'''
         if mol is None: mol = self.mol
-        user_set_breaksym = getattr(self, "init_guess_breaksym", None)
-        if user_set_breaksym is not None:
-            breaksym = user_set_breaksym
+        if breaksym is None: breaksym = self.init_guess_breaksym
         # For spin polarized system, no need to manually break spin symmetry
-        if mol.spin != 0:
-            breaksym = False
-        return init_guess_by_minao(mol, breaksym)
+        dm = hf.init_guess_by_minao(mol)
+        dma = dmb = dm*.5
+        dma, dmb = _break_dm_spin_symm(mol, (dma, dmb), breaksym)
+        return numpy.array((dma, dmb))
 
-    def init_guess_by_atom(self, mol=None, breaksym=BREAKSYM):
+    def init_guess_by_atom(self, mol=None, breaksym=None):
         if mol is None: mol = self.mol
-        user_set_breaksym = getattr(self, "init_guess_breaksym", None)
-        if user_set_breaksym is not None:
-            breaksym = user_set_breaksym
-        return init_guess_by_atom(mol, breaksym)
+        if breaksym is None: breaksym = self.init_guess_breaksym
+        dm = hf.init_guess_by_atom(mol)
+        dma = dmb = dm*.5
+        if mol.spin == 0 and breaksym:
+            if breaksym == 1:
+                #Add off-diagonal part for alpha DM
+                dma = mol.intor_symmetric('int1e_ovlp') * 1e-2
+                for b0, b1, p0, p1 in mol.aoslice_by_atom():
+                    dma[p0:p1,p0:p1] = dmb[p0:p1,p0:p1]
+            else:
+                dma, dmb = _break_dm_spin_symm(mol, (dma, dmb), breaksym)
+        return numpy.array((dma,dmb))
 
-    def init_guess_by_huckel(self, mol=None, breaksym=BREAKSYM):
+    def init_guess_by_huckel(self, mol=None, breaksym=None):
         if mol is None: mol = self.mol
-        user_set_breaksym = getattr(self, "init_guess_breaksym", None)
-        if user_set_breaksym is not None:
-            breaksym = user_set_breaksym
+        if breaksym is None: breaksym = self.init_guess_breaksym
         logger.info(self, 'Initial guess from on-the-fly Huckel, doi:10.1021/acs.jctc.8b01089.')
         mo_energy, mo_coeff = hf._init_guess_huckel_orbitals(mol, updated_rule = False)
         mo_energy = (mo_energy, mo_energy)
@@ -880,11 +885,9 @@ def init_guess_by_huckel(self, mol=None, breaksym=BREAKSYM):
             dma, dmb = _break_dm_spin_symm(mol, (dma, dmb))
         return numpy.array((dma,dmb))
 
-    def init_guess_by_mod_huckel(self, mol=None, breaksym=BREAKSYM):
+    def init_guess_by_mod_huckel(self, mol=None, breaksym=None):
         if mol is None: mol = self.mol
-        user_set_breaksym = getattr(self, "init_guess_breaksym", None)
-        if user_set_breaksym is not None:
-            breaksym = user_set_breaksym
+        if breaksym is None: breaksym = self.init_guess_breaksym
         logger.info(self, '''Initial guess from on-the-fly Huckel, doi:10.1021/acs.jctc.8b01089,
 employing the updated GWH rule from doi:10.1021/ja00480a005.''')
         mo_energy, mo_coeff = hf._init_guess_huckel_orbitals(mol, updated_rule = True)
@@ -893,14 +896,12 @@ def init_guess_by_mod_huckel(self, mol=None, breaksym=BREAKSYM):
         mo_occ = self.get_occ(mo_energy, mo_coeff)
         dma, dmb = self.make_rdm1(mo_coeff, mo_occ)
         if breaksym:
-            dma, dmb = _break_dm_spin_symm(mol, (dma, dmb))
+            dma, dmb = _break_dm_spin_symm(mol, (dma, dmb), breaksym)
         return numpy.array((dma,dmb))
 
-    def init_guess_by_1e(self, mol=None, breaksym=BREAKSYM):
+    def init_guess_by_1e(self, mol=None, breaksym=None):
         if mol is None: mol = self.mol
-        user_set_breaksym = getattr(self, "init_guess_breaksym", None)
-        if user_set_breaksym is not None:
-            breaksym = user_set_breaksym
+        if breaksym is None: breaksym = self.init_guess_breaksym
         logger.info(self, 'Initial guess from hcore.')
         h1e = self.get_hcore(mol)
         s1e = self.get_ovlp(mol)
@@ -911,12 +912,13 @@ def init_guess_by_1e(self, mol=None, breaksym=BREAKSYM):
         dma, dmb = self.make_rdm1(mo_coeff, mo_occ)
         natm = getattr(mol, 'natm', 0)  # handle custom Hamiltonian
         if natm > 0 and breaksym:
-            dma, dmb = _break_dm_spin_symm(mol, (dma, dmb))
+            dma, dmb = _break_dm_spin_symm(mol, (dma, dmb), breaksym)
         return numpy.array((dma,dmb))
 
-    def init_guess_by_sap(self, mol=None, breaksym=BREAKSYM, **kwargs):
+    def init_guess_by_sap(self, mol=None, breaksym=None, **kwargs):
         from pyscf.gto.basis import load
         if mol is None: mol = self.mol
+        if breaksym is None: breaksym = self.init_guess_breaksym
         sap_basis = self.sap_basis
         logger.info(self, '''Initial guess from superposition of atomic potentials (doi:10.1021/acs.jctc.8b01089)
 This is the Gaussian fit version as described in doi:10.1063/5.0004046.''')
@@ -935,11 +937,11 @@ def init_guess_by_sap(self, mol=None, breaksym=BREAKSYM, **kwargs):
             for key in sap_basis:
                 sapbas[key] = numpy.asarray(sap_basis[key][0][1:], dtype=float)
         else:
-            logger.error(self, 'sap_basis is of an unexpected datatype.')
-        user_set_breaksym = getattr(self, "init_guess_breaksym", None)
-        if user_set_breaksym is not None:
-            breaksym = user_set_breaksym
-        return init_guess_by_sap(mol, sap_basis=sapbas, breaksym=breaksym)
+            raise RuntimeError('sap_basis is of an unexpected datatype.')
+        dm = hf.init_guess_by_sap(mol, sapbas)
+        dma = dmb = dm*.5
+        dma, dmb = _break_dm_spin_symm(mol, (dma, dmb), breaksym)
+        return numpy.array((dma,dmb))
 
     def init_guess_by_chkfile(self, chkfile=None, project=None):
         if chkfile is None: chkfile = self.chkfile
@@ -1124,4 +1126,4 @@ class HF1e(UHF):
     def spin_square(self, mo_coeff=None, s=None):
         return .75, 2
 
-del (WITH_META_LOWDIN, PRE_ORTH_METHOD, BREAKSYM)
+del (WITH_META_LOWDIN, PRE_ORTH_METHOD)


Test patch:
diff --git a/pyscf/scf/test/test_uhf.py b/pyscf/scf/test/test_uhf.py
index addba13ea1..a60501eb16 100644
--- a/pyscf/scf/test/test_uhf.py
+++ b/pyscf/scf/test/test_uhf.py
@@ -70,10 +70,14 @@ def tearDownModule():
 
 class KnownValues(unittest.TestCase):
     def test_init_guess_minao(self):
+        mf = scf.UHF(mol)
         dm1 = mf.init_guess_by_minao(mol, breaksym=False)
         self.assertAlmostEqual(abs(dm1).sum(), 13.649710173723337, 9)
         dm2 = scf.uhf.get_init_guess(mol, key='minao')
         self.assertAlmostEqual(abs(dm2).sum(), 12.913908927027279, 9)
+        mf.init_guess_breaksym = 2
+        dm1 = mf.init_guess_by_minao(mol)
+        self.assertAlmostEqual(abs(dm1).sum(), 13.649710173723337, 9)
 
     def test_init_guess_1e(self):
         dm1 = scf.uhf.init_guess_by_1e(mol, breaksym=False)