Repository: beetbox/beets

Issue:
Lyrics plugin error "object of type 'NoneType' has no len()"
### Problem

`beet lyrics` was left running for a while, worked fine until it stopped at some unidentified song that triggered:

```
Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.7/bin/beet", line 8, in <module>
    sys.exit(main())
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/beets/ui/__init__.py", line 1291, in main
    _raw_main(args)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/beets/ui/__init__.py", line 1278, in _raw_main
    subcommand.func(lib, suboptions, subargs)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/beetsplug/lyrics.py", line 792, in func
    opts.force_refetch or self.config['force'],
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/beetsplug/lyrics.py", line 889, in fetch_item_lyrics
    lyrics = [self.get_lyrics(artist, title) for title in titles]
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/beetsplug/lyrics.py", line 889, in <listcomp>
    lyrics = [self.get_lyrics(artist, title) for title in titles]
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/beetsplug/lyrics.py", line 922, in get_lyrics
    lyrics = backend.fetch(artist, title)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/beetsplug/lyrics.py", line 347, in fetch
    self.fetch_url(hit["result"]["url"]))
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/beetsplug/lyrics.py", line 376, in _scrape_lyrics_from_html
    html = BeautifulSoup(html, "html.parser")
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/bs4/__init__.py", line 310, in __init__
    elif len(markup) <= 256 and (
TypeError: object of type 'NoneType' has no len()
```

### Setup

* OS: macOS High Sierra
* Python version: 3.7
* beets version: 1.5.0

Relevant issue: https://github.com/beetbox/beets/issues/479


Discussion:
No one else has this? Should I run some version so that it doesn't happen?

```
lyrics:
    auto: yes
    force: yes
    sources: genius musixmatch lyricwiki google
    google_API_key: redacted
    fallback: ''
    import.write: yes
```
No, this seems to be an actual bug, the fix from #480 apparently either got lost or didn't apply to all lyrics sources in the first place. I'll have a try at fixing this.

This probably didn't show up before because it only manifests when the lyrics provider's server is unreachable. If your connection is down entirely, that would show up much quicker.

Resolving patch:
diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py
--- a/beetsplug/lyrics.py
+++ b/beetsplug/lyrics.py
@@ -33,7 +33,8 @@
 from six.moves import urllib
 
 try:
-    from bs4 import SoupStrainer, BeautifulSoup
+    import bs4
+    from bs4 import SoupStrainer
     HAS_BEAUTIFUL_SOUP = True
 except ImportError:
     HAS_BEAUTIFUL_SOUP = False
@@ -219,7 +220,20 @@ def slug(text):
     return re.sub(r'\W+', '-', unidecode(text).lower().strip()).strip('-')
 
 
+if HAS_BEAUTIFUL_SOUP:
+    def try_parse_html(html, **kwargs):
+        try:
+            return bs4.BeautifulSoup(html, 'html.parser', **kwargs)
+        except HTMLParseError:
+            return None
+else:
+    def try_parse_html(html, **kwargs):
+        return None
+
+
 class Backend(object):
+    REQUIRES_BS = False
+
     def __init__(self, config, log):
         self._log = log
 
@@ -257,6 +271,7 @@ def fetch_url(self, url):
             return r.text
         else:
             self._log.debug(u'failed to fetch: {0} ({1})', url, r.status_code)
+            return None
 
     def fetch(self, artist, title):
         raise NotImplementedError()
@@ -286,11 +301,11 @@ def fetch(self, artist, title):
 
         html = self.fetch_url(url)
         if not html:
-            return
+            return None
         if "We detected that your IP is blocked" in html:
             self._log.warning(u'we are blocked at MusixMatch: url %s failed'
                               % url)
-            return
+            return None
         html_parts = html.split('<p class="mxm-lyrics__content')
         # Sometimes lyrics come in 2 or more parts
         lyrics_parts = []
@@ -302,10 +317,10 @@ def fetch(self, artist, title):
         # missing songs. this seems to happen after being blocked
         # above, when filling in the CAPTCHA.
         if "Instant lyrics for all your music." in lyrics:
-            return
+            return None
         # sometimes there are non-existent lyrics with some content
         if 'Lyrics | Musixmatch' in lyrics:
-            return
+            return None
         return lyrics
 
 
@@ -316,6 +331,8 @@ class Genius(Backend):
     bigishdata.com/2016/09/27/getting-song-lyrics-from-geniuss-api-scraping/
     """
 
+    REQUIRES_BS = True
+
     base_url = "https://api.genius.com"
 
     def __init__(self, config, log):
@@ -343,11 +360,14 @@ def fetch(self, artist, title):
             hit_artist = hit["result"]["primary_artist"]["name"]
 
             if slug(hit_artist) == slug(artist):
-                return self._scrape_lyrics_from_html(
-                    self.fetch_url(hit["result"]["url"]))
+                html = self.fetch_url(hit["result"]["url"])
+                if not html:
+                    return None
+                return self._scrape_lyrics_from_html(html)
 
         self._log.debug(u'Genius failed to find a matching artist for \'{0}\'',
                         artist)
+        return None
 
     def _search(self, artist, title):
         """Searches the genius api for a given artist and title
@@ -373,22 +393,24 @@ def _search(self, artist, title):
     def _scrape_lyrics_from_html(self, html):
         """Scrape lyrics from a given genius.com html"""
 
-        html = BeautifulSoup(html, "html.parser")
+        soup = try_parse_html(html)
+        if not soup:
+            return
 
         # Remove script tags that they put in the middle of the lyrics.
-        [h.extract() for h in html('script')]
+        [h.extract() for h in soup('script')]
 
         # Most of the time, the page contains a div with class="lyrics" where
         # all of the lyrics can be found already correctly formatted
         # Sometimes, though, it packages the lyrics into separate divs, most
         # likely for easier ad placement
-        lyrics_div = html.find("div", class_="lyrics")
+        lyrics_div = soup.find("div", class_="lyrics")
         if not lyrics_div:
             self._log.debug(u'Received unusual song page html')
-            verse_div = html.find("div",
+            verse_div = soup.find("div",
                                   class_=re.compile("Lyrics__Container"))
             if not verse_div:
-                if html.find("div",
+                if soup.find("div",
                              class_=re.compile("LyricsPlaceholder__Message"),
                              string="This song is an instrumental"):
                     self._log.debug('Detected instrumental')
@@ -410,6 +432,7 @@ def _scrape_lyrics_from_html(self, html):
 
 class Tekstowo(Backend):
     # Fetch lyrics from Tekstowo.pl.
+    REQUIRES_BS = True
 
     BASE_URL = 'http://www.tekstowo.pl'
     URL_PATTERN = BASE_URL + '/wyszukaj.html?search-title=%s&search-artist=%s'
@@ -417,27 +440,27 @@ class Tekstowo(Backend):
     def fetch(self, artist, title):
         url = self.build_url(title, artist)
         search_results = self.fetch_url(url)
+        if not search_results:
+            return None
         song_page_url = self.parse_search_results(search_results)
 
         if not song_page_url:
             return None
 
         song_page_html = self.fetch_url(song_page_url)
+        if not song_page_html:
+            return None
         return self.extract_lyrics(song_page_html)
 
     def parse_search_results(self, html):
-        if not HAS_BEAUTIFUL_SOUP:
-            return None
-
         html = _scrape_strip_cruft(html)
         html = _scrape_merge_paragraphs(html)
 
-        try:
-            html = BeautifulSoup(html, "html.parser")
-        except HTMLParseError:
+        soup = try_parse_html(html)
+        if not soup:
             return None
 
-        song_rows = html.find("div", class_="content"). \
+        song_rows = soup.find("div", class_="content"). \
             find("div", class_="card"). \
             find_all("div", class_="box-przeboje")
 
@@ -456,12 +479,11 @@ def extract_lyrics(self, html):
         html = _scrape_strip_cruft(html)
         html = _scrape_merge_paragraphs(html)
 
-        try:
-            html = BeautifulSoup(html, "html.parser")
-        except HTMLParseError:
+        soup = try_parse_html(html)
+        if not soup:
             return None
 
-        return html.find("div", class_="song-text").get_text()
+        return soup.find("div", class_="song-text").get_text()
 
 
 def remove_credits(text):
@@ -507,12 +529,6 @@ def scrape_lyrics_from_html(html):
     """Scrape lyrics from a URL. If no lyrics can be found, return None
     instead.
     """
-    if not HAS_BEAUTIFUL_SOUP:
-        return None
-
-    if not html:
-        return None
-
     def is_text_notcode(text):
         length = len(text)
         return (length > 20 and
@@ -522,10 +538,8 @@ def is_text_notcode(text):
     html = _scrape_merge_paragraphs(html)
 
     # extract all long text blocks that are not code
-    try:
-        soup = BeautifulSoup(html, "html.parser",
-                             parse_only=SoupStrainer(text=is_text_notcode))
-    except HTMLParseError:
+    soup = try_parse_html(html, parse_only=SoupStrainer(text=is_text_notcode))
+    if not soup:
         return None
 
     # Get the longest text element (if any).
@@ -539,6 +553,8 @@ def is_text_notcode(text):
 class Google(Backend):
     """Fetch lyrics from Google search results."""
 
+    REQUIRES_BS = True
+
     def __init__(self, config, log):
         super(Google, self).__init__(config, log)
         self.api_key = config['google_API_key'].as_str()
@@ -645,6 +661,8 @@ def fetch(self, artist, title):
                                               title, artist):
                     continue
                 html = self.fetch_url(url_link)
+                if not html:
+                    continue
                 lyrics = scrape_lyrics_from_html(html)
                 if not lyrics:
                     continue
@@ -654,10 +672,11 @@ def fetch(self, artist, title):
                                     item['displayLink'])
                     return lyrics
 
+        return None
+
 
 class LyricsPlugin(plugins.BeetsPlugin):
     SOURCES = ['google', 'musixmatch', 'genius', 'tekstowo']
-    BS_SOURCES = ['google', 'genius', 'tekstowo']
     SOURCE_BACKENDS = {
         'google': Google,
         'musixmatch': MusiXmatch,
@@ -727,15 +746,17 @@ def __init__(self):
                          for source in sources]
 
     def sanitize_bs_sources(self, sources):
-        for source in self.BS_SOURCES:
-            if source in sources:
+        enabled_sources = []
+        for source in sources:
+            if source.REQUIRES_BS:
                 self._log.debug(u'To use the %s lyrics source, you must '
                                 u'install the beautifulsoup4 module. See '
                                 u'the documentation for further details.'
                                 % source)
-                sources.remove(source)
+            else:
+                enabled_sources.append(source)
 
-        return sources
+        return enabled_sources
 
     def get_bing_access_token(self):
         params = {


Test patch: