Repository: mampfes/hacs_waste_collection_schedule

Issue:
Error "fetch failed for source RH Entsorgung"
Hi, since yesterday i get the following error:

> `Logger: waste_collection_schedule.scraper
> Source: custom_components/waste_collection_schedule/waste_collection_schedule/scraper.py:124
> Integration: waste_collection_schedule ([documentation](https://github.com/mampfes/hacs_waste_collection_schedule#readme))
> First occurred: May 9, 2022, 12:14:44 (2 occurrences)
> Last logged: 01:42:20
> 
> fetch failed for source RH Entsorgung: Traceback (most recent call last): File "/config/custom_components/waste_collection_schedule/waste_collection_schedule/scraper.py", line 122, in fetch entries = self._source.fetch() File "/config/custom_components/waste_collection_schedule/waste_collection_schedule/source/rh_entsorgung_de.py", line 137, in fetch date_parser.feed(r.text) File "/usr/local/lib/python3.9/html/parser.py", line 110, in feed self.goahead(0) File "/usr/local/lib/python3.9/html/parser.py", line 170, in goahead k = self.parse_starttag(i) File "/usr/local/lib/python3.9/html/parser.py", line 344, in parse_starttag self.handle_starttag(tag, attrs) File "/config/custom_components/waste_collection_schedule/waste_collection_schedule/source/rh_entsorgung_de.py", line 61, in handle_starttag if str(d["class"]).lower() == "work": KeyError: 'class'`

Could someone please help me? Is it an error on my side or from the plugin?


Discussion:

Resolving patch:
diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/source/rh_entsorgung_de.py b/custom_components/waste_collection_schedule/waste_collection_schedule/source/rh_entsorgung_de.py
--- a/custom_components/waste_collection_schedule/waste_collection_schedule/source/rh_entsorgung_de.py
+++ b/custom_components/waste_collection_schedule/waste_collection_schedule/source/rh_entsorgung_de.py
@@ -1,9 +1,8 @@
-import re
-from datetime import date
 from html.parser import HTMLParser
 
 import requests
 from waste_collection_schedule import Collection  # type: ignore[attr-defined]
+from waste_collection_schedule.service.ICS import ICS
 
 TITLE = "RH Entsorgung"
 DESCRIPTION = "Source for RHE (Rhein Hunsrück Entsorgung)."
@@ -40,70 +39,24 @@ def handle_starttag(self, tag, attrs):
                 self._args[d["name"]] = d["value"] if "value" in d else ""
 
 
-class CollectionParser(HTMLParser):
-    def __init__(self) -> None:
-        super().__init__()
-        self._entries: list[Collection] = []
-        self._current_type: str = None
-        self._capture_type: bool = False
-        self._capture_date: bool = False
-        self._date_pattern = re.compile(
-            r"(?P<day>\d{2})\.(?P<month>\d{2})\.(?P<year>\d{4})"
-        )
-
-    @property
-    def entries(self):
-        return self._entries
-
-    def handle_starttag(self, tag: str, attrs) -> None:
-        if tag == "p":
-            d = dict(attrs)
-            if str(d["class"]).lower() == "work":
-                self._capture_type = True
-        if self._current_type is not None and tag == "td":
-            d = dict(attrs)
-            if ("class" in d) and ("dia_c_abfuhrdatum" in str(d["class"])):
-                self._capture_date = True
-
-    def handle_data(self, data: str) -> None:
-        if self._capture_type:
-            self._current_type = data
-        if self._capture_date:
-            match = self._date_pattern.match(data)
-            self._entries.append(
-                Collection(
-                    date(int(match.group(3)), int(match.group(2)), int(match.group(1))),
-                    self._current_type,
-                )
-            )
-
-    def handle_endtag(self, tag: str) -> None:
-        if tag == "p" and self._capture_type:
-            self._capture_type = False
-        if tag == "td" and self._capture_date:
-            self._capture_date = False
-
-
 class Source:
     def __init__(
-        self,
-        city: str,
-        street: str,
-        house_number: int,
-        address_suffix: str = "",
-        garbage_types: list[int] = [1, 2, 3, 4, 5],
+        self, city: str, street: str, house_number: int, address_suffix: str = ""
     ):
         self._city = city
         self._street = street
         self._hnr = house_number
         self._suffix = address_suffix
-        self._garbage_types = garbage_types
+        self._ics = ICS()
 
     def fetch(self):
-        r = requests.get(
+        session = requests.session()
+
+        r = session.get(
             "https://aao.rh-entsorgung.de/WasteManagementRheinhunsrueck/WasteManagementServlet",
             params={"SubmitAction": "wasteDisposalServices", "InFrameMode": "TRUE"},
         )
+        r.raise_for_status()
         r.encoding = "utf-8"
 
         parser = HiddenInputParser()
@@ -114,25 +67,36 @@ def fetch(self):
         args["Strasse"] = self._street
         args["Hausnummer"] = str(self._hnr)
         args["Hausnummerzusatz"] = self._suffix
-        args["Zeitraum"] = "Die Leerungen der nächsten 3 Monate"
-        args["SubmitAction"] = "forward"
-        for type in range(1, 6):
-            args[f"ContainerGewaehlt_{type}"] = (
-                "on" if type in self._garbage_types else "off"
-            )
+        args["SubmitAction"] = "CITYCHANGED"
+        r = session.post(
+            "https://aao.rh-entsorgung.de/WasteManagementRheinhunsrueck/WasteManagementServlet",
+            data=args,
+        )
+        r.raise_for_status()
 
-        # First request returns wrong city. has to be called twice!
-        r = requests.post(
+        args["SubmitAction"] = "forward"
+        args["ContainerGewaehltRM"] = "on"
+        args["ContainerGewaehltBM"] = "on"
+        args["ContainerGewaehltLVP"] = "on"
+        args["ContainerGewaehltPA"] = "on"
+        args["ContainerGewaehltPrMuell"] = "on"
+        r = session.post(
             "https://aao.rh-entsorgung.de/WasteManagementRheinhunsrueck/WasteManagementServlet",
             data=args,
         )
+        r.raise_for_status()
 
-        r = requests.post(
+        args["ApplicationName"] = "com.athos.kd.rheinhunsrueck.AbfuhrTerminModel"
+        args["SubmitAction"] = "filedownload_ICAL"
+        r = session.post(
             "https://aao.rh-entsorgung.de/WasteManagementRheinhunsrueck/WasteManagementServlet",
             data=args,
         )
-        r.encoding = "utf-8"
+        r.raise_for_status()
+
+        dates = self._ics.convert(r.text)
 
-        date_parser = CollectionParser()
-        date_parser.feed(r.text)
-        return date_parser.entries
+        entries = []
+        for d in dates:
+            entries.append(Collection(d[0], d[1]))
+        return entries


Test patch: