Repository: mne-tools/mne-python

Issue:
mne.time_frequency.psd_welch add median option
I want to robustly estimate a power spectrum from raw data. 
I am concerned that segments of my raw data contain artifacts. 
mne.time_frequency.psd_welch divides the raw data into N (possibly overlapping) windows, computes N psds, and returns the average psd. 
Could we add the option to return the median psd? 
By this I mean for each frequency bin we have N estimates of the power (from the N psds), and we return the median of these N values. 
I believe the R function welchPSD does this. 



Discussion:
+1 for a method = "mean" | "median" as the R implem.

do you want to take a stab at implementing this?

Would it make sense to also make `method` accept a float between .5 and 1, in order to indicate the percentile of data from which the average is computed? `1` would be equivalent to the mean, `.5` would be equivalent to the median, and the rest would be trimmed means, which in my experience are more stable than median in our low-sampled datasets.

(also see https://github.com/mne-tools/mne-python/issues/3044)

yes I think that trimmed mean would be nice.

See API of scipy:

http://docs.scipy.org/doc/scipy-0.17.0/reference/generated/scipy.stats.trim_mean.html

let's try to copy it as much as possible.

I will take a stab at implementing. 
I agree a trimmed mean method is best. 
After looking at the code in more detail, I believe mne calls scipy.signal.welch to compute the psd, so it is scipy that I should attempt to contribute to first (to get scipy.signal.welch to return a trimmed mean).

@sapttrs do you have time to look into this? If not I can try

If we go this rout consider the discussion on the other recent issue about
Annotations. We then should also think of conputing the welch epochs
explicitly to be able to compute stats across windows, e.g, for estimating
variability/confidence in spectral features.
On Fri, 18 Nov 2016 at 21:51, Eric Larson notifications@github.com wrote:

> @sapttrs https://github.com/sapttrs do you have time to look into this?
> If not I can try
> 
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> https://github.com/mne-tools/mne-python/issues/3336#issuecomment-261638651,
> or mute the thread
> https://github.com/notifications/unsubscribe-auth/AB0fiqscuNG0EBiNjOwB44j28ftshaLoks5q_g_dgaJpZM4I96T0
> .

@Eric89GXL I haven't had time to look into it, but a student at our lab has worked on it. I will get an update from him.
@Eric89GXL @sapttrs scipy's welch function (scipy/signal/spectral.py) can be easily modified to implement trimmed mean. However, mne's psd_welch (mne/time_frequency/psd.py) uses parallel calls to welch to compute the psds, so the way to do it seems to be by modifying `psds' at the end of _psd_welch.

By adding a parameter tmean with default argument = 0 to both psd_welch and _psd_welch, the following code added to the end of _psd_welch seems to do the job. It can be modified to support parallel processing as well.

   

    if tmean > 0:
        from scipy import stats
        median_window = np.zeros(psds.shape[1])   
        for j in range(psds.shape[1]): 
            # process all spectral densities at element j of each window.
            median_window[j] = stats.trim_mean(psds[:,j], tmean)
        return median_window, psds, freqs                       
    else:
        return psds, freqs
@fkamel is the median computed over channels or over windows? From what I understand you want it over windows
Marking this as easy now that we have https://github.com/scipy/scipy/pull/8970

Resolving patch:
diff --git a/doc/conf.py b/doc/conf.py
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -584,6 +584,7 @@ def reset_warnings(gallery_conf, fname):
     'n_splits', 'n_scores', 'n_outputs', 'n_trials', 'n_estimators', 'n_tasks',
     'nd_features', 'n_classes', 'n_targets', 'n_slices', 'n_hpi', 'n_fids',
     'n_elp', 'n_pts', 'n_tris', 'n_nodes', 'n_nonzero', 'n_events_out',
+    'n_segments',
     # Undocumented (on purpose)
     'RawKIT', 'RawEximia', 'RawEGI', 'RawEEGLAB', 'RawEDF', 'RawCTF', 'RawBTi',
     'RawBrainVision', 'RawCurry',
diff --git a/mne/time_frequency/psd.py b/mne/time_frequency/psd.py
--- a/mne/time_frequency/psd.py
+++ b/mne/time_frequency/psd.py
@@ -10,10 +10,24 @@
 from .multitaper import psd_array_multitaper
 
 
-def _psd_func(epoch, noverlap, n_per_seg, nfft, fs, freq_mask, func):
+def _spect_func(epoch, n_overlap, n_per_seg, nfft, fs, freq_mask, func):
     """Aux function."""
-    return func(epoch, fs=fs, nperseg=n_per_seg, noverlap=noverlap,
-                nfft=nfft, window='hamming')[2][..., freq_mask, :]
+    _, _,  spect = func(epoch, fs=fs, nperseg=n_per_seg, noverlap=n_overlap,
+                        nfft=nfft, window='hamming')
+    return spect[..., freq_mask, :]
+
+
+def _welch_func(epoch, n_overlap, n_per_seg, nfft, fs, freq_mask, average,
+                func):
+    """Aux function."""
+    kws = dict(fs=fs, nperseg=n_per_seg, noverlap=n_overlap, nfft=nfft,
+               window='hamming', average=average)
+
+    if average == 'mean':  # Compatibility with SciPy <1.2
+        del kws['average']
+
+    _, psd = func(epoch, **kws)
+    return psd[..., freq_mask]
 
 
 def _check_nfft(n, n_fft, n_per_seg, n_overlap):
@@ -62,7 +76,7 @@ def _check_psd_data(inst, tmin, tmax, picks, proj, reject_by_annotation=False):
 
 @verbose
 def psd_array_welch(x, sfreq, fmin=0, fmax=np.inf, n_fft=256, n_overlap=0,
-                    n_per_seg=None, n_jobs=1, verbose=None):
+                    n_per_seg=None, n_jobs=1, average='mean', verbose=None):
     """Compute power spectral density (PSD) using Welch's method.
 
     Parameters
@@ -85,13 +99,24 @@ def psd_array_welch(x, sfreq, fmin=0, fmax=np.inf, n_fft=256, n_overlap=0,
         Length of each Welch segment (windowed with a Hamming window). Defaults
         to None, which sets n_per_seg equal to n_fft.
     %(n_jobs)s
+    average : str | None
+        How to average the segments. If ``mean`` (default), calculate the
+        arithmetic mean. If ``median``, calculate the median, corrected for
+        its bias relative to the mean. If ``None``, returns the unaggregated
+        segments.
+
+        .. versionadded:: 0.19.0
     %(verbose)s
 
     Returns
     -------
-    psds : ndarray, shape (..., n_freqs) or
-        The power spectral densities. All dimensions up to the last will
-        be the same as input.
+    psds : ndarray, shape (..., n_freqs) or (..., n_freqs, n_segments)
+        The power spectral densities. If ``average='mean`` or
+        ``average='median'``, the returned array will have the same shape
+        as the input data plus an additional frequency dimension.
+        If ``average=None``, the returned array will have the same shape as
+        the input data plus two additional dimensions corresponding to
+        frequencies and the unaggregated segments, respectively.
     freqs : ndarray, shape (n_freqs,)
         The frequencies.
 
@@ -99,7 +124,6 @@ def psd_array_welch(x, sfreq, fmin=0, fmax=np.inf, n_fft=256, n_overlap=0,
     -----
     .. versionadded:: 0.14.0
     """
-    from scipy.signal import spectrogram
     dshape = x.shape[:-1]
     n_times = x.shape[-1]
     x = x.reshape(-1, n_times)
@@ -114,24 +138,43 @@ def psd_array_welch(x, sfreq, fmin=0, fmax=np.inf, n_fft=256, n_overlap=0,
     freqs = freqs[freq_mask]
 
     # Parallelize across first N-1 dimensions
-    parallel, my_psd_func, n_jobs = parallel_func(_psd_func, n_jobs=n_jobs)
     x_splits = np.array_split(x, n_jobs)
-    f_spectrogram = parallel(my_psd_func(d, noverlap=n_overlap, nfft=n_fft,
+
+    if average in ['mean', 'median']:
+        from scipy.signal import welch
+        parallel, my_welch_func, n_jobs = parallel_func(_welch_func,
+                                                        n_jobs=n_jobs)
+
+        psds = parallel(my_welch_func(d, fs=sfreq, freq_mask=freq_mask,
+                                      n_per_seg=n_per_seg, n_overlap=n_overlap,
+                                      nfft=n_fft, average=average, func=welch)
+                        for d in x_splits)
+        psds = np.concatenate(psds, axis=0)
+        psds.shape = dshape + (-1,)
+    elif average is None:
+        from scipy.signal import spectrogram
+        parallel, my_spect_func, n_jobs = parallel_func(_spect_func,
+                                                        n_jobs=n_jobs)
+
+        f_spect = parallel(my_spect_func(d, n_overlap=n_overlap,
+                                         nfft=n_fft,
                                          fs=sfreq, freq_mask=freq_mask,
-                                         func=spectrogram, n_per_seg=n_per_seg)
-                             for d in x_splits)
+                                         func=spectrogram,
+                                         n_per_seg=n_per_seg)
+                           for d in x_splits)
+        psds = np.concatenate(f_spect, axis=0)
+        psds.shape = dshape + (len(freqs), -1)
+    else:
+        raise ValueError('average must be one of `mean`, `median`, or None, '
+                         'got {}'.format(average))
 
-    # Combining, reducing windows and reshaping to original data shape
-    psds = np.concatenate([np.nanmean(f_s, axis=-1)
-                           for f_s in f_spectrogram], axis=0)
-    psds.shape = dshape + (-1,)
     return psds, freqs
 
 
 @verbose
 def psd_welch(inst, fmin=0, fmax=np.inf, tmin=None, tmax=None, n_fft=256,
               n_overlap=0, n_per_seg=None, picks=None, proj=False, n_jobs=1,
-              reject_by_annotation=True, verbose=None):
+              reject_by_annotation=True, average='mean', verbose=None):
     """Compute the power spectral density (PSD) using Welch's method.
 
     Calculates periodograms for a sliding window over the time dimension, then
@@ -171,14 +214,24 @@ def psd_welch(inst, fmin=0, fmax=np.inf, tmin=None, tmax=None, n_fft=256,
         Evoked object. Defaults to True.
 
         .. versionadded:: 0.15.0
+    average : str | None
+        How to average the segments. If ``mean`` (default), calculate the
+        arithmetic mean. If ``median``, calculate the median, corrected for
+        its bias relative to the mean. If ``None``, returns the unaggregated
+        segments.
+
+        .. versionadded:: 0.19.0
     %(verbose)s
 
     Returns
     -------
-    psds : ndarray, shape (..., n_freqs)
-        The power spectral densities. If input is of type Raw,
-        then psds will be shape (n_channels, n_freqs), if input is type Epochs
-        then psds will be shape (n_epochs, n_channels, n_freqs).
+    psds : ndarray, shape (..., n_freqs) or (..., n_freqs, n_segments)
+        The power spectral densities. If ``average='mean`` or
+        ``average='median'`` and input is of type Raw or Evoked, then psds will
+        be of shape (n_channels, n_freqs); if input is of type Epochs, then
+        psds will be of shape (n_epochs, n_channels, n_freqs).
+        If ``average=None``, the returned array will have an additional
+        dimension corresponding to the unaggregated segments.
     freqs : ndarray, shape (n_freqs,)
         The frequencies.
 
@@ -198,7 +251,7 @@ def psd_welch(inst, fmin=0, fmax=np.inf, tmin=None, tmax=None, n_fft=256,
                                   reject_by_annotation=reject_by_annotation)
     return psd_array_welch(data, sfreq, fmin=fmin, fmax=fmax, n_fft=n_fft,
                            n_overlap=n_overlap, n_per_seg=n_per_seg,
-                           n_jobs=n_jobs, verbose=verbose)
+                           average=average, n_jobs=n_jobs, verbose=verbose)
 
 
 @verbose
diff --git a/tutorials/time-freq/plot_sensors_time_frequency.py b/tutorials/time-freq/plot_sensors_time_frequency.py
--- a/tutorials/time-freq/plot_sensors_time_frequency.py
+++ b/tutorials/time-freq/plot_sensors_time_frequency.py
@@ -13,6 +13,7 @@
 """  # noqa: E501
 # Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
 #          Stefan Appelhoff <stefan.appelhoff@mailbox.org>
+#          Richard Höchenberger <richard.hoechenberger@gmail.com>
 #
 # License: BSD (3-clause)
 import os.path as op
@@ -21,7 +22,7 @@
 import matplotlib.pyplot as plt
 
 import mne
-from mne.time_frequency import tfr_morlet, psd_multitaper
+from mne.time_frequency import tfr_morlet, psd_multitaper, psd_welch
 from mne.datasets import somato
 
 ###############################################################################
@@ -46,7 +47,7 @@
                     baseline=baseline, reject=dict(grad=4000e-13, eog=350e-6),
                     preload=True)
 
-epochs.resample(150., npad='auto')  # resample to reduce computation time
+epochs.resample(200., npad='auto')  # resample to reduce computation time
 
 ###############################################################################
 # Frequency analysis
@@ -78,10 +79,51 @@
 ax.plot(freqs, psds_mean, color='k')
 ax.fill_between(freqs, psds_mean - psds_std, psds_mean + psds_std,
                 color='k', alpha=.5)
-ax.set(title='Multitaper PSD (gradiometers)', xlabel='Frequency',
+ax.set(title='Multitaper PSD (gradiometers)', xlabel='Frequency (Hz)',
        ylabel='Power Spectral Density (dB)')
 plt.show()
 
+###############################################################################
+# Notably, :func:`mne.time_frequency.psd_welch` supports the keyword argument
+# ``average``, which specifies how to estimate the PSD based on the individual
+# windowed segments. The default is ``average='mean'``, which simply calculates
+# the arithmetic mean across segments. Specifying ``average='median'``, in
+# contrast, returns the PSD based on the median of the segments (corrected for
+# bias relative to the mean), which is a more robust measure.
+
+# Estimate PSDs based on "mean" and "median" averaging for comparison.
+kwargs = dict(fmin=2, fmax=40, n_jobs=1)
+psds_welch_mean, freqs_mean = psd_welch(epochs, average='mean', **kwargs)
+psds_welch_median, freqs_median = psd_welch(epochs, average='median', **kwargs)
+
+# Convert power to dB scale.
+psds_welch_mean = 10 * np.log10(psds_welch_mean)
+psds_welch_median = 10 * np.log10(psds_welch_median)
+
+# We will only plot the PSD for a single sensor in the first epoch.
+ch_name = 'MEG 0122'
+ch_idx = epochs.info['ch_names'].index(ch_name)
+epo_idx = 0
+
+_, ax = plt.subplots()
+ax.plot(freqs_mean, psds_welch_mean[epo_idx, ch_idx, :], color='k',
+        ls='-', label='mean of segments')
+ax.plot(freqs_median, psds_welch_median[epo_idx, ch_idx, :], color='k',
+        ls='--', label='median of segments')
+
+ax.set(title='Welch PSD ({}, Epoch {})'.format(ch_name, epo_idx),
+       xlabel='Frequency (Hz)', ylabel='Power Spectral Density (dB)')
+ax.legend(loc='upper right')
+plt.show()
+
+###############################################################################
+# Lastly, we can also retrieve the unaggregated segments by passing
+# ``average=None`` to :func:`mne.time_frequency.psd_welch`. The dimensions of
+# the returned array are ``(n_epochs, n_sensors, n_freqs, n_segments)``.
+
+psds_welch_unagg, freqs_unagg = psd_welch(epochs, average=None, **kwargs)
+print(psds_welch_unagg.shape)
+
 ###############################################################################
 # .. _inter-trial-coherence:
 #


Test patch:
diff --git a/mne/time_frequency/tests/test_psd.py b/mne/time_frequency/tests/test_psd.py
--- a/mne/time_frequency/tests/test_psd.py
+++ b/mne/time_frequency/tests/test_psd.py
@@ -6,7 +6,7 @@
 
 from mne import pick_types, Epochs, read_events
 from mne.io import RawArray, read_raw_fif
-from mne.utils import run_tests_if_main
+from mne.utils import run_tests_if_main, requires_version
 from mne.time_frequency import psd_welch, psd_multitaper, psd_array_welch
 
 base_dir = op.join(op.dirname(__file__), '..', '..', 'io', 'tests', 'data')
@@ -160,6 +160,69 @@ def test_psd():
         assert (psds_ev.shape == (len(kws['picks']), len(freqs)))
 
 
+@requires_version('scipy', '1.2.0')
+@pytest.mark.parametrize('kind', ('raw', 'epochs', 'evoked'))
+def test_psd_welch_average_kwarg(kind):
+    """Test `average` kwarg of psd_welch()."""
+    raw = read_raw_fif(raw_fname)
+    picks_psd = [0, 1]
+
+    # Populate raw with sinusoids
+    rng = np.random.RandomState(40)
+    data = 0.1 * rng.randn(len(raw.ch_names), raw.n_times)
+    freqs_sig = [8., 50.]
+    for ix, freq in zip(picks_psd, freqs_sig):
+        data[ix, :] += 2 * np.sin(np.pi * 2. * freq * raw.times)
+    first_samp = raw._first_samps[0]
+    raw = RawArray(data, raw.info)
+
+    tmin, tmax = -0.5, 0.5
+    fmin, fmax = 0, np.inf
+    n_fft = 256
+    n_per_seg = 128
+    n_overlap = 0
+
+    event_id = 2
+    events = read_events(event_fname)
+    events[:, 0] -= first_samp
+
+    kws = dict(fmin=fmin, fmax=fmax, tmin=tmin, tmax=tmax, n_fft=n_fft,
+               n_per_seg=n_per_seg, n_overlap=n_overlap, picks=picks_psd)
+
+    if kind == 'raw':
+        inst = raw
+    elif kind == 'epochs':
+        inst = Epochs(raw, events[:10], event_id, tmin, tmax, picks=picks_psd,
+                      proj=False, preload=True, baseline=None)
+    elif kind == 'evoked':
+        inst = Epochs(raw, events[:10], event_id, tmin, tmax, picks=picks_psd,
+                      proj=False, preload=True, baseline=None).average()
+    else:
+        raise ValueError('Unknown parametrization passed to test, check test '
+                         'for typos.')
+
+    psds_mean, freqs_mean = psd_welch(inst=inst, average='mean', **kws)
+    psds_median, freqs_median = psd_welch(inst=inst, average='median', **kws)
+    psds_unagg, freqs_unagg = psd_welch(inst=inst, average=None, **kws)
+
+    # Frequencies should be equal across all "average" types, as we feed in
+    # the exact same data.
+    assert_allclose(freqs_mean, freqs_median)
+    assert_allclose(freqs_mean, freqs_unagg)
+
+    # For `average=None`, the last dimension contains the un-aggregated
+    # segments.
+    assert psds_mean.shape == psds_median.shape
+    assert psds_mean.shape == psds_unagg.shape[:-1]
+    assert_allclose(psds_mean, psds_unagg.mean(axis=-1))
+
+    # SciPy's welch() function corrects the median PSD for its bias relative to
+    # the mean.
+    from scipy.signal.spectral import _median_bias
+    median_bias = _median_bias(psds_unagg.shape[-1])
+    assert_allclose(psds_median, np.median(psds_unagg, axis=-1) / median_bias)
+
+
 @pytest.mark.slowtest
 def test_compares_psd():
     """Test PSD estimation on raw for plt.psd and scipy.signal.welch."""
@@ -204,4 +267,5 @@ def test_compares_psd():
     assert (np.sum(psds_welch < 0) == 0)
     assert (np.sum(psds_mpl < 0) == 0)
 
+
 run_tests_if_main()