Repository: mampfes/hacs_waste_collection_schedule

Issue:
[Request] AWB Esslingen
Hi there,

I tried to integrate AWB Esslingen into the Waste Collection Schedule locally which seems to use abfall.io, but I couldn't extract the information needed.

https://www.awb-es.de/abfuhr/abfuhrtermine/__Abfuhrtermine-suchen.html

Are you able to extract the needed information? I couldn't find the id's and so on. Seems like they hid the requests somehow server-side to me. Or am I missing something?

I could just get to the point where they show the ics file like [here](https://api.abfall.io/?kh=DaA02103019b46345f1998698563DaAd&t=ics&s=09bfb229c9154cd3f75316e2813a9075)

Thanks in advance,

psi


Discussion:

Resolving patch:
diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/source/awb_es_de.py b/custom_components/waste_collection_schedule/waste_collection_schedule/source/awb_es_de.py
new file mode 100644
--- /dev/null
+++ b/custom_components/waste_collection_schedule/waste_collection_schedule/source/awb_es_de.py
@@ -0,0 +1,59 @@
+import requests
+from bs4 import BeautifulSoup
+from waste_collection_schedule import Collection  # type: ignore[attr-defined]
+from waste_collection_schedule.service.ICS import ICS
+
+TITLE = "Abfallwirtschaftsbetrieb Esslingen"
+DESCRIPTION = "Source for AWB Esslingen, Germany"
+URL = "https://www.awb-es.de"
+
+TEST_CASES = {
+    "Aichwald": {"city": "Aichwald", "street": "Alte Dorfstrasse"},
+    "Kohlberg": {"city": "Kohlberg", "street": "alle Straßen"},
+}
+
+HEADERS = {"user-agent": "Mozilla/5.0 (xxxx Windows NT 10.0; Win64; x64)"}
+
+
+class Source:
+    def __init__(self, city, street=None):
+        self._city = city
+        self._street = street
+        self._ics = ICS()
+
+    def fetch(self):
+        session = requests.Session()
+
+        params = {
+            "city": self._city,
+            "street": self._street,
+            "direct": "true",
+        }
+        r = session.get(
+            "https://www.awb-es.de/abfuhr/abfuhrtermine/__Abfuhrtermine.html",
+            params=params,
+        )
+        r.raise_for_status()
+
+        soup = BeautifulSoup(r.text, features="html.parser")
+        downloads = soup.find_all("a", href=True)
+        ics_url = None
+        for download in downloads:
+            href = download.get("href")
+            if "t=ics" in href:
+                ics_url = href
+
+        if ics_url is None:
+            raise Exception(f"ics url not found")
+
+        # get ics file
+        r = session.get(ics_url, headers=HEADERS)
+        r.raise_for_status()
+
+        # parse ics file
+        dates = self._ics.convert(r.text)
+
+        entries = []
+        for d in dates:
+            entries.append(Collection(d[0], d[1]))
+        return entries
diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/source/awb_oldenburg_de.py b/custom_components/waste_collection_schedule/waste_collection_schedule/source/awb_oldenburg_de.py
--- a/custom_components/waste_collection_schedule/waste_collection_schedule/source/awb_oldenburg_de.py
+++ b/custom_components/waste_collection_schedule/waste_collection_schedule/source/awb_oldenburg_de.py
@@ -1,9 +1,8 @@
-import requests
-from waste_collection_schedule import Collection  # type: ignore[attr-defined]
-
 import urllib
-import icalendar
 
+import requests
+from waste_collection_schedule import Collection  # type: ignore[attr-defined]
+from waste_collection_schedule.service.ICS import ICS
 
 TITLE = "AWB Oldenburg"
 DESCRIPTION = "Source for 'Abfallwirtschaftsbetrieb Stadt Oldenburg (Oldb)'."
@@ -17,19 +16,22 @@ class Source:
     def __init__(self, street, house_number):
         self._street = street
         self._house_number = house_number
+        self._ics = ICS(regex=r"(.*)\:\s*\!")
 
     def fetch(self):
 
         args = {
-            'id': 430,
-            'tx_citkoabfall_abfallkalender[strasse]': str(self._street).encode('utf-8'),
-            'tx_citkoabfall_abfallkalender[hausnummer]': str(self._house_number).encode('utf-8'),
-            'tx_citkoabfall_abfallkalender[abfallarten][0]': 61,
-            'tx_citkoabfall_abfallkalender[abfallarten][1]': 60,
-            'tx_citkoabfall_abfallkalender[abfallarten][2]': 59,
-            'tx_citkoabfall_abfallkalender[abfallarten][3]': 58,
-            'tx_citkoabfall_abfallkalender[action]': 'ics',
-            'tx_citkoabfall_abfallkalender[controller]': 'FrontendIcs'
+            "id": 430,
+            "tx_citkoabfall_abfallkalender[strasse]": str(self._street).encode("utf-8"),
+            "tx_citkoabfall_abfallkalender[hausnummer]": str(self._house_number).encode(
+                "utf-8"
+            ),
+            "tx_citkoabfall_abfallkalender[abfallarten][0]": 61,
+            "tx_citkoabfall_abfallkalender[abfallarten][1]": 60,
+            "tx_citkoabfall_abfallkalender[abfallarten][2]": 59,
+            "tx_citkoabfall_abfallkalender[abfallarten][3]": 58,
+            "tx_citkoabfall_abfallkalender[action]": "ics",
+            "tx_citkoabfall_abfallkalender[controller]": "FrontendIcs",
         }
 
         # use '%20' instead of '+' in URL
@@ -37,18 +39,11 @@ def fetch(self):
         args = urllib.parse.urlencode(args, quote_via=urllib.parse.quote)
 
         # post request
-        reply = requests.get(URL, params=args)
+        r = requests.get(URL, params=args)
 
-        # create calender from reply
-        gcal = icalendar.Calendar.from_ical(reply.text)
+        dates = self._ics.convert(r.text)
 
-        # iterate over events and add to waste collection
         entries = []
-        for component in gcal.walk():
-            if component.name == "VEVENT":
-                type = component.get('summary')
-                start_time = component.get('dtstart').dt
-
-                entries.append(Collection(start_time.date(), type))
-
+        for d in dates:
+            entries.append(Collection(d[0], d[1]))
         return entries


Test patch: