Repository: mne-tools/mne-python

Issue:
Explained variance of PCs
Is there an easy way to get the percentage of explained variance for each of the principal components?



Discussion:
no but it would indeed be a useful infomation to store in a proj object.

~~combining #1999:~~

> ~~plot_projs_topomap defaults to the Vectorview-all if a layout isn't specified. plot_ica_components uses a neat little helper function _prepare_topo_plot to get the position information. I think this would be a good alternative, but the downside is that the proj objects don't have an info attribute. if it had this, it would make it easy for non-Vectorview systems.~~

adding an info to proj files is possible but a mess with historical files.

is passing info to the function so annoying?

why not have the proj's added to whatever you are calculating it on average, raw, etc. then just plot the projections from that object. i.e. mne.plot_proj(raw), which has the info?

I realize with already have a plot_projs_topomap in the ProjMixin so you
can do:

evoked.plot_projs_topomap()
epochs.plot_projs_topomap()
raw.plot_projs_topomap()

ok cool no mod of `proj` for `info`. does `proj` store the diagonal matrix anywhere, I'm assuming that its eigenvalues would give the percentage of explained variance

no it does not store it so far.

can this be determined from the S matrix in the SVD? if I remember my linear algebra, the diagonal should contain the eigenvalues in ranked order. to get the explained variance, their sum should be scaled to one.

you seem to be on the right track @teonlamont ;) PR welcome

this may help you: https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/decomposition/pca.py#L284 :)

@jasmainak, thanks for the link.

a few questions have come up:

~~1. is there a reason why the [computed covariance](https://github.com/mne-tools/mne-python/blob/master/mne/proj.py#L160) isn't mean-centered?~~
1. from the looks of things `data` that goes into [_compute_proj](https://github.com/mne-tools/mne-python/blob/master/mne/proj.py#L71). it's already the covariance at this point, so its SVD should be `U * S**2 * Ut`. is this correct?
2. should this be a separate function of an add-on the existing ones? It easily falls out of the computation. Otherwise, the PCA will be recalculated for the explained variance function.

EEG and MEG data are commonly assumed to be zero mean simply because you
apply baseline correction or high pass filtering. ok?


Resolving patch:
diff --git a/mne/cov.py b/mne/cov.py
--- a/mne/cov.py
+++ b/mne/cov.py
@@ -424,7 +424,7 @@ def compute_raw_covariance(raw, tmin=None, tmax=None, tstep=0.2,
     info = pick_info(raw.info, picks)
     idx_by_type = channel_indices_by_type(info)
 
-    # Read data in chuncks
+    # Read data in chunks
     for first in range(start, stop, step):
         last = first + step
         if last >= stop:
diff --git a/mne/io/proj.py b/mne/io/proj.py
--- a/mne/io/proj.py
+++ b/mne/io/proj.py
@@ -341,6 +341,12 @@ def _read_proj(fid, node, verbose=None):
         else:
             active = False
 
+        tag = find_tag(fid, item, FIFF.FIFF_MNE_ICA_PCA_EXPLAINED_VAR)
+        if tag is not None:
+            explained_var = tag.data
+        else:
+            explained_var = None
+
         # handle the case when data is transposed for some reason
         if data.shape[0] == len(names) and data.shape[1] == nvec:
             data = data.T
@@ -352,7 +358,8 @@ def _read_proj(fid, node, verbose=None):
         #   Use exactly the same fields in data as in a named matrix
         one = Projection(kind=kind, active=active, desc=desc,
                          data=dict(nrow=nvec, ncol=nchan, row_names=None,
-                                   col_names=names, data=data))
+                                   col_names=names, data=data),
+                         explained_var=explained_var)
 
         projs.append(one)
 
@@ -401,6 +408,9 @@ def _write_proj(fid, projs):
         write_int(fid, FIFF.FIFF_MNE_PROJ_ITEM_ACTIVE, proj['active'])
         write_float_matrix(fid, FIFF.FIFF_PROJ_ITEM_VECTORS,
                            proj['data']['data'])
+        if proj['explained_var'] is not None:
+            write_float(fid, FIFF.FIFF_MNE_ICA_PCA_EXPLAINED_VAR,
+                        proj['explained_var'])
         end_block(fid, FIFF.FIFFB_PROJ_ITEM)
 
     end_block(fid, FIFF.FIFFB_PROJ)
@@ -627,11 +637,13 @@ def make_eeg_average_ref_proj(info, activate=True, verbose=None):
         raise ValueError('Cannot create EEG average reference projector '
                          '(no EEG data found)')
     vec = np.ones((1, n_eeg)) / n_eeg
+    explained_var = None
     eeg_proj_data = dict(col_names=eeg_names, row_names=None,
                          data=vec, nrow=1, ncol=n_eeg)
     eeg_proj = Projection(active=activate, data=eeg_proj_data,
                           desc='Average EEG reference',
-                          kind=FIFF.FIFFV_MNE_PROJ_ITEM_EEG_AVREF)
+                          kind=FIFF.FIFFV_MNE_PROJ_ITEM_EEG_AVREF,
+                          explained_var=explained_var)
     return eeg_proj
 
 
diff --git a/mne/proj.py b/mne/proj.py
--- a/mne/proj.py
+++ b/mne/proj.py
@@ -97,15 +97,19 @@ def _compute_proj(data, info, n_grad, n_mag, n_eeg, desc_prefix, verbose=None):
         if n == 0:
             continue
         data_ind = data[ind][:, ind]
-        U = linalg.svd(data_ind, full_matrices=False,
-                       overwrite_a=True)[0][:, :n]
-        for k, u in enumerate(U.T):
+        # data is the covariance matrix: U * S**2 * Ut
+        U, Sexp2, _ = linalg.svd(data_ind, full_matrices=False,
+                                 overwrite_a=True)
+        U = U[:, :n]
+        exp_var = Sexp2 / Sexp2.sum()
+        exp_var = exp_var[:n]
+        for k, (u, var) in enumerate(zip(U.T, exp_var)):
             proj_data = dict(col_names=names, row_names=None,
                              data=u[np.newaxis, :], nrow=1, ncol=u.size)
             this_desc = "%s-%s-PCA-%02d" % (desc, desc_prefix, k + 1)
             logger.info("Adding projection: %s" % this_desc)
             proj = Projection(active=False, data=proj_data,
-                              desc=this_desc, kind=1)
+                              desc=this_desc, kind=1, explained_var=var)
             projs.append(proj)
 
     return projs


Test patch:
diff --git a/mne/preprocessing/tests/test_ssp.py b/mne/preprocessing/tests/test_ssp.py
--- a/mne/preprocessing/tests/test_ssp.py
+++ b/mne/preprocessing/tests/test_ssp.py
@@ -34,6 +34,17 @@ def test_compute_proj_ecg():
         # heart rate at least 0.5 Hz, but less than 3 Hz
         assert_true(events.shape[0] > 0.5 * dur_use and
                     events.shape[0] < 3 * dur_use)
+        ssp_ecg = [proj for proj in projs if proj['desc'].startswith('ECG')]
+        # check that the first principal component have a certain minimum
+        ssp_ecg = [proj for proj in ssp_ecg if 'PCA-01' in proj['desc']]
+        thresh_eeg, thresh_axial, thresh_planar = .9, .3, .1
+        for proj in ssp_ecg:
+            if 'planar' in proj['desc']:
+                assert_true(proj['explained_var'] > thresh_planar)
+            elif 'axial' in proj['desc']:
+                assert_true(proj['explained_var'] > thresh_axial)
+            elif 'eeg' in proj['desc']:
+                assert_true(proj['explained_var'] > thresh_eeg)
         # XXX: better tests
 
         # without setting a bad channel, this should throw a warning
@@ -62,6 +73,17 @@ def test_compute_proj_eog():
         assert_true(len(projs) == (7 + n_projs_init))
         assert_true(np.abs(events.shape[0] -
                     np.sum(np.less(eog_times, dur_use))) <= 1)
+        ssp_eog = [proj for proj in projs if proj['desc'].startswith('EOG')]
+        # check that the first principal component have a certain minimum
+        ssp_eog = [proj for proj in ssp_eog if 'PCA-01' in proj['desc']]
+        thresh_eeg, thresh_axial, thresh_planar = .9, .3, .1
+        for proj in ssp_eog:
+            if 'planar' in proj['desc']:
+                assert_true(proj['explained_var'] > thresh_planar)
+            elif 'axial' in proj['desc']:
+                assert_true(proj['explained_var'] > thresh_axial)
+            elif 'eeg' in proj['desc']:
+                assert_true(proj['explained_var'] > thresh_eeg)
         # XXX: better tests
 
         # This will throw a warning b/c simplefilter('always')
diff --git a/mne/tests/test_proj.py b/mne/tests/test_proj.py
--- a/mne/tests/test_proj.py
+++ b/mne/tests/test_proj.py
@@ -122,6 +122,9 @@ def test_compute_proj_epochs():
                 p2_data = p2_data[:, mask]
             corr = np.corrcoef(p1_data, p2_data)[0, 1]
             assert_array_almost_equal(corr, 1.0, 5)
+            if p2['explained_var']:
+                assert_array_almost_equal(p1['explained_var'],
+                                          p2['explained_var'])
 
     # test that you can compute the projection matrix
     projs = activate_proj(projs)