Repository: mne-tools/mne-python

Issue:
Fix the cited_mne.py script to update "Publications by users" page
In our docs, we link to a [Publications by users](http://martinos.org/mne/stable/cited.html) page, which is being created by our [`cited_mne.py`](https://github.com/mne-tools/mne-python/blob/master/doc/sphinxext/cited_mne.py) script, authored by @jasmainak (see https://github.com/mne-tools/mne-python/issues/1935 and https://github.com/mne-tools/mne-python/pull/2954 )

This script has not been run in a long time and according to @larsoner, it's not trivial to get it to work again (python 2 and several dead dependencies).

Some volunteer should overhaul `cited_mne.py` so that we can update our **Publications by users** and show the world how great MNE-Python is.

(the alternative is a stale page in the docs, last updated 2017)

I have not yet found a trivial workflow used by others ... or a package that can easily be tweaked / copied over.

cc: @jona-sassenhagen who tried this before


Discussion:
I would not spend time on this and point to the scholar page :)


Resolving patch:
diff --git a/doc/sphinxext/cited_mne.py b/doc/sphinxext/cited_mne.py
deleted file mode 100755
--- a/doc/sphinxext/cited_mne.py
+++ /dev/null
@@ -1,261 +0,0 @@
-#!/usr/bin/env python
-"""Parse google scholar -> rst for MNE citations.
-
-Example usage::
-
-    $ ./cited_mne.py --backend selenium --clear
-
-This requires joblib, BeautifulSoup, and selenium.
-selenium in turn requires geckodriver:
-
-    https://github.com/mozilla/geckodriver/releases
-
-The process will involve window popups to satisfy
-CAPTCHA checks.
-"""
-
-# Author: Mainak Jas <mainak.jas@telecom-paristech.fr>
-# License : BSD 3-clause
-
-# Parts of this code were copied from google_scholar_parser
-# (https://github.com/carlosp420/google_scholar_parser)
-
-import os
-import os.path as op
-import re
-import time
-import random
-import requests
-
-import numpy as np
-from joblib import Memory
-from BeautifulSoup import BeautifulSoup
-
-from mne.externals.tempita import Template
-from mne.commands.utils import get_optparser
-
-# cache to avoid making too many calls to Google Scholar
-cachedir = 'cachedir'
-if not os.path.exists(cachedir):
-    os.mkdir(cachedir)
-mem = Memory(cachedir=cachedir, verbose=2)
-
-UA = ('Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.9.2.9) '
-      'Gecko/20100913 Firefox/3.6.9')
-
-# ##### Templates for citations #####
-html = (u""":orphan:
-
-.. _cited:
-
-Publications by users
-=====================
-
-Papers citing MNE (%d) as extracted from Google Scholar (on %s).
-
-""")
-
-cite_template = Template(u"""
-{{for ii, publication in enumerate(publications)}}
-{{ii + 1}}. {{publication}}.
-{{endfor}}
-
-""")
-
-
-def parse_soup_page(soup):
-    """Parse the page using BeautifulSoup.
-
-    Parameters
-    ----------
-    soup : instance of BeautifulSoup
-        The page to be parsed.
-
-    Returns
-    -------
-    titles : list
-        The article titles.
-    authors : list
-        The name of the authors.
-    links : list
-        Hyperlinks to the articles.
-    """
-    titles, authors, links = list(), list(), list()
-    for div in soup.findAll('div'):
-        if div.name == "div" and div.get('class') == "gs_ri":
-            links.append(div.a['href'])
-            div_pub = div.findAll('div')
-            for d in div_pub:
-                if d.name == 'div' and d.get('class') == 'gs_a':
-                    authors.append(d.text)
-            titles.append(div.a.text)
-    return titles, authors, links
-
-
-def get_total_citations(soup):
-    """Get total citations."""
-    results = soup.find('div', attrs={'id': 'gs_ab_md'}).contents[0]
-    matches = re.search(r"About\s(\d+)\s", results)
-    if matches:
-        hits = matches.groups()[0]
-        return hits
-
-
-def _get_soup(url, backend='selenium'):
-    """Get BeautifulSoup object from url.
-
-    Parameters
-    ----------
-    url : str
-        The url to fetch.
-    backend : 'selenium' | 'requests'
-        Use selenium by default because google can ask for captcha. For
-        'selenium' backend Firefox must be installed.
-
-    Returns
-    -------
-    soup : instance of BeautifulSoup
-        The soup page from the url.
-    """
-    if backend == 'requests':
-        req = requests.get(url, headers={'User-Agent': UA})
-        html_doc = req.text
-        soup = BeautifulSoup(html_doc)
-        if soup.find('div', attrs={'id': 'gs_ab_md'}) is None:
-            print('Falling back on to selenium backend due to captcha.')
-            backend = 'selenium'
-
-    if backend == 'selenium':
-        from selenium import webdriver
-        import selenium.webdriver.support.ui as ui
-
-        driver = webdriver.Firefox()
-        # give enough time to solve captcha.
-        wait = ui.WebDriverWait(driver, 200)
-
-        driver.get(url)
-        wait.until(lambda driver: driver.find_elements_by_id('gs_ab_md'))
-
-        html_doc = driver.page_source
-        soup = BeautifulSoup(html_doc)
-        driver.close()
-
-    return soup
-
-
-@mem.cache
-def get_citing_articles(cites_url, backend):
-    """Get the citing articles.
-
-    Parameters
-    ----------
-    cites_url: str
-        A citation url from Google Scholar.
-    backend : 'selenium' | 'requests'
-        Use selenium by default because google can ask for captcha. For
-        'selenium' backend Firefox must be installed.
-
-
-    Returns
-    -------
-    titles : list
-        The article titles.
-    authors : list
-        The name of the authors.
-    links : list
-        Hyperlinks to the articles.
-    """
-    n = random.random() * 5
-    time.sleep(n)
-    print("\nSleeping: {0} seconds".format(n))
-
-    # GS seems to allow only 20 hits per page!
-    cites_url += "&num=20"
-    soup = _get_soup(cites_url, backend=backend)
-    hits = get_total_citations(soup)
-    print("Got a total of {0} citations".format(hits))
-
-    hits = int(hits)
-    index = 0
-    titles, authors, links = list(), list(), list()
-    while hits > 1:
-        n = random.random() * 2
-        time.sleep(n)
-        if index > 0:
-            url = cites_url + "&start=" + str(index)
-        else:
-            url = cites_url
-        index += 20
-        hits -= 20
-        print("{0} more citations to process".format(hits))
-        soup = soup = _get_soup(url, backend=backend)
-        title, author, link = parse_soup_page(soup)
-        for this_title, this_author, this_link in zip(title, author, link):
-            titles.append(this_title)
-            authors.append(this_author)
-            links.append(this_link)
-
-    return titles, authors, links
-
-if __name__ == '__main__':
-    parser = get_optparser(__file__)
-    parser.add_option("-c", "--clear", dest="clear", action='store_true',
-                      help="if True, clear the cache.", default=False)
-    parser.add_option("-b", "--backend", dest="backend",
-                      help="backend for parsing (selenium | requests)",
-                      default='requests')
-    options, args = parser.parse_args()
-    backend, clear = options.backend, options.clear
-
-    if clear:
-        mem.clear()
-
-    random.seed()
-    gen_date = time.strftime("%B %d, %Y")
-    url_tails = ['1521584321377182930', '12188330066413208874']
-    papers = ['MEG and EEG data analysis with MNE-Python',
-              'MNE software for processing MEG and EEG data']
-
-    publications = list()
-    for url_tail, paper in zip(url_tails, papers):
-        titles, authors, links = get_citing_articles(
-            'https://scholar.google.co.in/scholar?cites=%s'
-            % url_tail, backend=backend)
-
-        this_publication = list()
-        for ii in range(len(titles)):
-            pub = '`%s. <%s>`_. %s' % (titles[ii], links[ii], authors[ii])
-            this_publication.append(pub)
-
-        this_publication = [p.encode('utf8') for p in this_publication]
-        publications.append(this_publication)
-
-    # get a union of the citations for the two papers, sorted in
-    # alphabetic order
-    publications = np.union1d(publications[1], publications[0]).tolist()
-
-    html = html % (len(publications), gen_date)
-
-    # sort by year of publication
-    years = list()
-    for pub in publications:
-        m = re.search(r'\d{4} -', pub)
-        if m is None:
-            years.append(-1)
-        else:
-            years.append(int(m.group(0)[:-2]))
-    order = np.argsort(years)[::-1]
-    publications = [publications[idx] for idx in order]
-
-    # filter out publications not containing (http://, https://, ftp://)
-    publications = [p for p in publications if
-                    any(sub in p for sub in ('http://', 'https://', 'ftp://'))]
-
-    # create rst & cleanup
-    this_html = cite_template.substitute(publications=publications)
-    this_html = this_html.replace('&hellip;', '...')
-    html += this_html
-
-    # output an rst file
-    with open(op.join('..', 'cited.rst'), 'w') as f:
-        f.write(html.encode('utf8'))


Test patch: