Repository: sopel-irc/sopel

Issue:
[etymology] Site HTML changes broke etymology module
etymonline no longer uses nice, clean `dt`/`dd` markup, sadly. There will need to be a little work done to rewrite the parser in `etymology.py` so it can pull etymologies from the new HTML structure.


Discussion:

Resolving patch:
diff --git a/sopel/modules/etymology.py b/sopel/modules/etymology.py
--- a/sopel/modules/etymology.py
+++ b/sopel/modules/etymology.py
@@ -2,45 +2,28 @@
 """
 etymology.py - Sopel Etymology Module
 Copyright 2007-9, Sean B. Palmer, inamidst.com
+Copyright 2018-9, Sopel contributors
 Licensed under the Eiffel Forum License 2.
 
 https://sopel.chat
 """
 from __future__ import unicode_literals, absolute_import, print_function, division
 
+from re import sub
+from requests import get
+from sopel.module import commands, example, NOLIMIT
 try:
-    from html import unescape
-except ImportError:
+    # Python 2.6-2.7
     from HTMLParser import HTMLParser
-
-    # pep8 dictates a blank line here...
-    def unescape(s):
-        return HTMLParser.unescape.__func__(HTMLParser, s)
-import re
-import requests
-from sopel.module import commands, example, NOLIMIT
-
-etyuri = 'http://etymonline.com/?term=%s'
-etysearch = 'http://etymonline.com/?search=%s'
-
-r_definition = re.compile(r'(?ims)<dd[^>]*>.*?</dd>')
-r_tag = re.compile(r'<(?!!)[^>]+>')
-r_whitespace = re.compile(r'[\t\r\n ]+')
-
-abbrs = [
-    'cf', 'lit', 'etc', 'Ger', 'Du', 'Skt', 'Rus', 'Eng', 'Amer.Eng', 'Sp',
-    'Fr', 'N', 'E', 'S', 'W', 'L', 'Gen', 'J.C', 'dial', 'Gk',
-    '19c', '18c', '17c', '16c', 'St', 'Capt', 'obs', 'Jan', 'Feb', 'Mar',
-    'Apr', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec', 'c', 'tr', 'e', 'g'
-]
-t_sentence = r'^.*?(?<!%s)(?:\.(?= [A-Z0-9]|\Z)|\Z)'
-r_sentence = re.compile(t_sentence % ')(?<!'.join(abbrs))
+    h = HTMLParser()
+    unescape = h.unescape
+except ImportError:
+    # Python 3
+    from html import unescape  # https://stackoverflow.com/a/2087433
 
 
-def text(html):
-    html = r_tag.sub('', html)
-    html = r_whitespace.sub(' ', html)
-    return unescape(html).strip()
+ETYURI = 'https://www.etymonline.com/word/%s'
+ETYSEARCH = 'https://www.etymonline.com/search?q=%s'
 
 
 def etymology(word):
@@ -48,30 +31,30 @@ def etymology(word):
     # entries? - http://swhack.com/logs/2006-07-19#T15-05-29
 
     if len(word) > 25:
-        raise ValueError("Word too long: %s[...]" % word[:10])
-    word = {'axe': 'ax/axe'}.get(word, word)
+        raise ValueError("Word too long: %s[…]" % word[:10])
 
-    bytes = requests.get(etyuri % word).text
-    definitions = r_definition.findall(bytes)
-
-    if not definitions:
+    ety = get(ETYURI % word)
+    if ety.status_code != 200:
         return None
 
-    defn = text(definitions[0])
-    m = r_sentence.match(defn)
-    if not m:
-        return None
-    sentence = m.group(0)
+    # Let's find it
+    start = ety.text.find("word__defination")
+    start = ety.text.find("<p>", start)
+    stop = ety.text.find("</p>", start)
+    sentence = ety.text[start + 3:stop]
+    # Clean up
+    sentence = unescape(sentence)
+    sentence = sub('<[^<]+?>', '', sentence)
 
     maxlength = 275
     if len(sentence) > maxlength:
         sentence = sentence[:maxlength]
         words = sentence[:-5].split(' ')
         words.pop()
-        sentence = ' '.join(words) + ' [...]'
+        sentence = ' '.join(words) + ' […]'
 
     sentence = '"' + sentence.replace('"', "'") + '"'
-    return sentence + ' - ' + (etyuri % word)
+    return sentence + ' - ' + (ETYURI % word)
 
 
 @commands('ety')
@@ -83,16 +66,18 @@ def f_etymology(bot, trigger):
     try:
         result = etymology(word)
     except IOError:
-        msg = "Can't connect to etymonline.com (%s)" % (etyuri % word)
+        msg = "Can't connect to etymonline.com (%s)" % (ETYURI % word)
         bot.msg(trigger.sender, msg)
         return NOLIMIT
     except (AttributeError, TypeError):
         result = None
+    except ValueError as ve:
+        result = str(ve)
 
     if result is not None:
         bot.msg(trigger.sender, result)
     else:
-        uri = etysearch % word
+        uri = ETYSEARCH % word
         msg = 'Can\'t find the etymology for "%s". Try %s' % (word, uri)
         bot.msg(trigger.sender, msg)
         return NOLIMIT


Test patch: