Task: Bug in NAO symmetry-averaged density matrix (issue #1755) (pyscf/pyscf#1803)

org: pyscf
repo: pyscf
pr_number: 1803
issue_title: Bug in NAO symmetry-averaged density matrix (issue #1755)
resolved_issues: number: 1755
title: Problem with lo.nao.nao function :  non-symmetric results for benzene
body: Hi,
I'm trying to get natural atomic orbital populations using both lo.orth_ao or lo.nao.nao functions and it appears there is an issue because I don't get symmetric results for a d6h benzene geometry at RHF level. When calling lo.orth_ao with Lowdin or meta_lowdin I do get symmetric enough results.
Just to confirm I was  operating correctly with the matrices, I've just tried your example at 
[https://github.com/pyscf/pyscf/blob/master/examples/local_orb/01-pop_with_nao.py](url)
but using the following geometry
mol.atom='''
 6       -0.000000   -1.399560    0.000000
 6       -1.212054   -0.699779   -0.000000
 6        1.212054   -0.699779   -0.000000
 1       -2.156901   -1.245293    0.000000
 1        2.156901   -1.245293    0.000000
 6       -1.212054    0.699779    0.000000
 6        1.212054    0.699779    0.000000
 1       -2.156901    1.245293   -0.000000
 1        2.156901    1.245293   -0.000000
 6       -0.000000    1.399560   -0.000000
 1        0.000000    2.490576    0.000000
 1        0.000000   -2.490576   -0.000000
'''

The results printed are as follows:
 ** Mulliken pop  **
pop of  0 C 1s        2.00000
pop of  0 C 2s        0.95998
pop of  0 C 3s        0.00097
pop of  0 C 2px       1.04921
pop of  0 C 2py       1.17158
pop of  0 C 2pz       0.99624
pop of  0 C 3px       0.00483
pop of  0 C 3py       0.00663
pop of  0 C 3pz       0.00130
pop of  0 C 3dxy      0.00736
pop of  0 C 3dyz      0.00134
pop of  0 C 3dz^2     0.00571
pop of  0 C 3dxz      0.00114
pop of  0 C 3dx2-y2    0.00332
pop of  1 C 1s        2.00000
pop of  1 C 2s        0.96229
pop of  1 C 3s        0.00096
pop of  1 C 2px       1.14762
pop of  1 C 2py       1.08598
pop of  1 C 2pz       1.00342
pop of  1 C 3px       0.00618
pop of  1 C 3py       0.00683
pop of  1 C 3pz       0.00086
pop of  1 C 3dxy      0.00434
pop of  1 C 3dyz      0.00119
pop of  1 C 3dz^2     0.00569
pop of  1 C 3dxz      0.00129
pop of  1 C 3dx2-y2    0.00630
pop of  2 C 1s        2.00000
pop of  2 C 2s        0.95762
pop of  2 C 3s        0.00099
pop of  2 C 2px       1.13327
pop of  2 C 2py       1.07281
pop of  2 C 2pz       0.98791
pop of  2 C 3px       0.00719
pop of  2 C 3py       0.00449
pop of  2 C 3pz       0.00265
pop of  2 C 3dxy      0.00432
pop of  2 C 3dyz      0.00120
pop of  2 C 3dz^2     0.00573
pop of  2 C 3dxz      0.00129
pop of  2 C 3dx2-y2    0.00640
pop of  3 H 1s        0.77763
pop of  3 H 2s        0.00206
pop of  3 H 2px       0.00411
pop of  3 H 2py       0.00208
pop of  3 H 2pz       0.00006
pop of  4 H 1s        0.78690
pop of  4 H 2s        0.00214
pop of  4 H 2px       0.00402
pop of  4 H 2py       0.00205
pop of  4 H 2pz       0.00006
pop of  5 C 1s        2.00000
pop of  5 C 2s        0.95762
pop of  5 C 3s        0.00099
pop of  5 C 2px       1.13327
pop of  5 C 2py       1.07281
pop of  5 C 2pz       0.98791
pop of  5 C 3px       0.00719
pop of  5 C 3py       0.00449
pop of  5 C 3pz       0.00265
pop of  5 C 3dxy      0.00432
pop of  5 C 3dyz      0.00120
pop of  5 C 3dz^2     0.00573
pop of  5 C 3dxz      0.00129
pop of  5 C 3dx2-y2    0.00640
pop of  6 C 1s        2.00000
pop of  6 C 2s        0.96229
pop of  6 C 3s        0.00096
pop of  6 C 2px       1.14762
pop of  6 C 2py       1.08598
pop of  6 C 2pz       1.00342
pop of  6 C 3px       0.00618
pop of  6 C 3py       0.00683
pop of  6 C 3pz       0.00086
pop of  6 C 3dxy      0.00434
pop of  6 C 3dyz      0.00119
pop of  6 C 3dz^2     0.00569
pop of  6 C 3dxz      0.00129
pop of  6 C 3dx2-y2    0.00630
pop of  7 H 1s        0.78690
pop of  7 H 2s        0.00214
pop of  7 H 2px       0.00402
pop of  7 H 2py       0.00205
pop of  7 H 2pz       0.00006
pop of  8 H 1s        0.77763
pop of  8 H 2s        0.00206
pop of  8 H 2px       0.00411
pop of  8 H 2py       0.00208
pop of  8 H 2pz       0.00006
pop of  9 C 1s        2.00000
pop of  9 C 2s        0.95998
pop of  9 C 3s        0.00097
pop of  9 C 2px       1.04921
pop of  9 C 2py       1.17158
pop of  9 C 2pz       0.99624
pop of  9 C 3px       0.00483
pop of  9 C 3py       0.00663
pop of  9 C 3pz       0.00130
pop of  9 C 3dxy      0.00736
pop of  9 C 3dyz      0.00134
pop of  9 C 3dz^2     0.00571
pop of  9 C 3dxz      0.00114
pop of  9 C 3dx2-y2    0.00332
pop of  10 H 1s        0.78221
pop of  10 H 2s        0.00210
pop of  10 H 2px       0.00106
pop of  10 H 2py       0.00507
pop of  10 H 2pz       0.00006
pop of  11 H 1s        0.78221
pop of  11 H 2s        0.00210
pop of  11 H 2px       0.00106
pop of  11 H 2py       0.00507
pop of  11 H 2pz       0.00006
 ** Mulliken atomic charges  **
charge of  0C =     -0.20959
charge of  1C =     -0.23295
charge of  2C =     -0.18585
charge of  3H =      0.21407
charge of  4H =      0.20482
charge of  5C =     -0.18585
charge of  6C =     -0.23295
charge of  7H =      0.20482
charge of  8H =      0.21407
charge of  9C =     -0.20959
charge of  10H =      0.20950
charge of  11H =      0.20950

The asymmetry of the C atoms is way too much.  It is already seen in the contribution of nao's 2s orbitals. Curiously, the C atoms in para position are equivalent to each other.

Thank you very much for your help.

Pedro


Reference solution:
fix_patch: diff --git a/pyscf/lo/nao.py b/pyscf/lo/nao.py
index cc5ba2c48d..3cf505cb30 100644
--- a/pyscf/lo/nao.py
+++ b/pyscf/lo/nao.py
@@ -23,7 +23,7 @@
 '''
 
 import sys
-from functools import reduce
+from functools import reduce, lru_cache
 import numpy
 import scipy.linalg
 from pyscf import lib
@@ -72,6 +72,11 @@ def nao(mol, mf, s=None, restore=True):
     pre_occ, pre_nao = _prenao_sub(mol, p, s)
     cnao = _nao_sub(mol, pre_occ, pre_nao)
     if restore:
+        if mol.cart:
+            # The atomic natural character for Cartesian basis is not clearly
+            # defined. Skip restore.
+            return cnao
+
         # restore natural character
         p_nao = reduce(numpy.dot, (cnao.T, p, cnao))
         s_nao = numpy.eye(p_nao.shape[0])
@@ -98,10 +103,12 @@ def _prenao_sub(mol, p, s):
 
             if mol.cart:
                 degen = (l + 1) * (l + 2) // 2
+                p_frag = _cart_average_mat(p, l, idx)
+                s_frag = _cart_average_mat(s, l, idx)
             else:
                 degen = l * 2 + 1
-            p_frag = _spheric_average_mat(p, l, idx, degen)
-            s_frag = _spheric_average_mat(s, l, idx, degen)
+                p_frag = _sph_average_mat(p, l, idx)
+                s_frag = _sph_average_mat(s, l, idx)
             e, v = scipy.linalg.eigh(p_frag, s_frag)
             e = e[::-1]
             v = v[:,::-1]
@@ -187,12 +194,24 @@ def _core_val_ryd_list(mol):
         count[ia,l] += nc
     return core_lst, val_lst, rydbg_lst
 
-def _spheric_average_mat(mat, l, lst, degen=None):
-    if degen is None:
-        degen = l * 2 + 1
-    nd = len(lst) // degen
-    mat_frag = mat[lst][:,lst].reshape(nd,degen,nd,degen)
-    return numpy.einsum('imjn->ij', mat_frag) / degen
+@lru_cache(10)
+def _cart_averge_wt(l):
+    '''Weight matrix for spherical symmetry averaging in Cartensian GTOs'''
+    c = mole.cart2sph(l, normalized='sp')
+    return numpy.einsum('pi,qi->pq', c, c)
+
+def _cart_average_mat(mat, l, lst):
+    degen = (l+1) * (l+2) // 2
+    nshl = len(lst) // degen
+    submat = mat[lst[:,None],lst].reshape(nshl,degen,nshl,degen)
+    wt = _cart_averge_wt(l)
+    return numpy.einsum('imjn,mn->ij', submat, wt) / (2*l+1)
+
+def _sph_average_mat(mat, l, lst):
+    degen = 2 * l + 1
+    nshl = len(lst) // degen
+    submat = mat[lst[:,None],lst].reshape(nshl,degen,nshl,degen)
+    return numpy.einsum('imjm->ij', submat) / (2*l+1)
 
 def set_atom_conf(element, description):
     '''Change the default atomic core and valence configuration to the one
diff --git a/pyscf/lo/orth.py b/pyscf/lo/orth.py
index 70e5c894a6..c463038fe1 100644
--- a/pyscf/lo/orth.py
+++ b/pyscf/lo/orth.py
@@ -331,28 +331,3 @@ def orth_ao(mf_or_mol, method=ORTH_METHOD, pre_orth_ao=REF_BASIS, s=None):
     return c_orth
 
 del (ORTH_METHOD)
-
-
-if __name__ == '__main__':
-    from pyscf import scf
-    from pyscf.lo import nao
-    mol = gto.Mole()
-    mol.verbose = 1
-    mol.output = 'out_orth'
-    mol.atom.extend([
-        ['O' , (0. , 0.     , 0.)],
-        [1   , (0. , -0.757 , 0.587)],
-        [1   , (0. , 0.757  , 0.587)] ])
-    mol.basis = {'H': '6-31g',
-                 'O': '6-31g',}
-    mol.build()
-
-    mf = scf.RHF(mol)
-    mf.scf()
-
-    c0 = nao.prenao(mol, mf.make_rdm1())
-    c = orth_ao(mol, 'meta_lowdin', c0)
-
-    s = mol.intor_symmetric('int1e_ovlp_sph')
-    p = reduce(numpy.dot, (s, mf.make_rdm1(), s))
-    print(reduce(numpy.dot, (c.T, p, c)).diagonal())