Repository: mampfes/hacs_waste_collection_schedule

Issue:
Berlin Recycling
[Berlin Recycling](berlin-recycling.de) is handling my paper waste. Unfortunately their collection dates are hidden behind a login and need to parsed from their HTML.

I found this Python implementation based on BeautifulSoup: https://github.com/belzetrigger/domoticz-BR/blob/master/brHelper.py

I'm not sure how easy it would be to integrate. I'd be happy to help with debugging and testing, but I fear my python isn't good enough to implement it myself.


Discussion:
Hi, yes - the example should help to create a source for waste_collection_schedule. This would be the first source that requires authentication. Anyway - adding a new source may take a while.

Resolving patch:
diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/source/berlin_recycling_de.py b/custom_components/waste_collection_schedule/waste_collection_schedule/source/berlin_recycling_de.py
new file mode 100644
--- /dev/null
+++ b/custom_components/waste_collection_schedule/waste_collection_schedule/source/berlin_recycling_de.py
@@ -0,0 +1,91 @@
+import json
+from datetime import datetime
+from html.parser import HTMLParser
+
+import requests
+from waste_collection_schedule import Collection  # type: ignore[attr-defined]
+
+TITLE = "Berline Recycling"
+DESCRIPTION = "Source for Berlin Recycling waste collection."
+URL = "https://berlin-recycling.de"
+TEST_CASES = {
+    "Germanenstrasse": {
+        "username": "!secret berlin_recycling_username",
+        "password": "!secret berlin_recycling_password",
+    },
+}
+
+
+# Parser for HTML input (hidden) text
+class HiddenInputParser(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self._args = {}
+
+    @property
+    def args(self):
+        return self._args
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "input":
+            d = dict(attrs)
+            if str(d["type"]).lower() == "hidden":
+                self._args[d["name"]] = d["value"] if "value" in d else ""
+
+
+SERVICE_URL = "https://kundenportal.berlin-recycling.de/"
+
+
+class Source:
+    def __init__(self, username, password):
+        self._username = username
+        self._password = password
+
+    def fetch(self):
+        session = requests.session()
+
+        # first get returns session specific url
+        r = session.get(SERVICE_URL, allow_redirects=False)
+
+        # get session id's
+        r = session.get(r.url)
+
+        parser = HiddenInputParser()
+        parser.feed(r.text)
+        args = parser.args
+        args["__EVENTTARGET"] = "btnLog"
+        args["__EVENTARGUMENT"] = None
+        args["Username"] = self._username
+        args["Password"] = self._password
+
+        # login
+        r = session.post(r.url, data=args)
+        serviceUrl = r.url
+
+        request_data = {"withhtml": "true"}
+        r = session.post(serviceUrl + "/GetDashboard", json=request_data)
+
+        request_data = {"datasettable": "ENWIS_ABFUHRKALENDER"}
+        r = session.post(serviceUrl + "/ChangeDatasetTable", json=request_data)
+
+        request_data = {
+            "datasettablecode": "ENWIS_ABFUHRKALENDER",
+            "startindex": 0,
+            "searchtext": "",
+            "rangefilter": "",
+            "ordername": "",
+            "orderdir": "",
+            "ClientParameters": "",
+            "headrecid": "",
+        }
+        r = session.post(serviceUrl + "/GetDatasetTableHead", json=request_data)
+
+        data = json.loads(r.text)
+        # load json again, because response is double coded
+        data = json.loads(data["d"])
+
+        entries = []
+        for d in data["data"]:
+            date = datetime.strptime(d["Task Date"], "%Y-%m-%d").date()
+            entries.append(Collection(date, d["Material Description"]))
+        return entries
diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/source/bsr_de.py b/custom_components/waste_collection_schedule/waste_collection_schedule/source/bsr_de.py
--- a/custom_components/waste_collection_schedule/waste_collection_schedule/source/bsr_de.py
+++ b/custom_components/waste_collection_schedule/waste_collection_schedule/source/bsr_de.py
@@ -6,7 +6,7 @@
 
 TITLE = "BSR"
 DESCRIPTION = "Source for Berliner Stadtreinigungsbetriebe waste collection."
-URL = "bsr.de"
+URL = "https://bsr.de"
 TEST_CASES = {
     "Bahnhofstr., 12159 Berlin (Tempelhof-Schöneberg)": {
         "abf_strasse": "Bahnhofstr., 12159 Berlin (Tempelhof-Schöneberg)",


Test patch:
diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/test/test_sources.py b/custom_components/waste_collection_schedule/waste_collection_schedule/test/test_sources.py
--- a/custom_components/waste_collection_schedule/waste_collection_schedule/test/test_sources.py
+++ b/custom_components/waste_collection_schedule/waste_collection_schedule/test/test_sources.py
@@ -2,10 +2,16 @@
 
 import argparse
 import importlib
+import re
 import site
 import traceback
 from pathlib import Path
 
+import yaml
+
+SECRET_FILENAME = "secrets.yaml"
+SECRET_REGEX = re.compile(r"!secret\s(\w+)")
+
 
 def main():
     parser = argparse.ArgumentParser(description="Test sources.")
@@ -17,6 +23,18 @@ def main():
     )
     args = parser.parse_args()
 
+    # read secrets.yaml
+    secrets = {}
+    try:
+        with open(SECRET_FILENAME) as stream:
+            try:
+                secrets = yaml.safe_load(stream)
+            except yaml.YAMLError as exc:
+                print(exc)
+    except FileNotFoundError:
+        # ignore missing secrets.yaml
+        pass
+
     package_dir = Path(__file__).resolve().parents[2]
     source_dir = package_dir / "waste_collection_schedule" / "source"
 
@@ -44,9 +62,19 @@ def main():
         assert "URL" in names
         assert "TEST_CASES" in names
 
-        # create source
+        # run through all test-cases
         for name, tc in module.TEST_CASES.items():
-            # run through all test-cases
+            # replace secrets in arguments
+            for key, value in tc.items():
+                match = SECRET_REGEX.fullmatch(value)
+                if match is not None:
+                    id = match.group(1)
+                    if id in secrets:
+                        tc[key] = secrets[id]
+                    else:
+                        print(f"identifier '{id}' not found in {SECRET_FILENAME}")
+
+            # create source
             source = module.Source(**tc)
             try:
                 result = source.fetch()