Repository: mampfes/hacs_waste_collection_schedule

Issue:
Request: AWB LK KA
Greetings,

It'd be great if you could add the AWB LK Karlsruhe to your list.
You can pull the events without registration from here: https://www.awb-landkreis-karlsruhe.de/start/wissen/abfuhrkalender.html

I'll try to use the ical option, but I'd like to see it working without, if it's possible.

Regards,
Felix


Discussion:
AWB LK Karlsruhe provides a "ical-Kalenderabo", therefore the ICS source is normally sufficient and we don't have to write a dedicated source.

Please let us know if ICS works for you.

Resolving patch:
diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/source/ics.py b/custom_components/waste_collection_schedule/waste_collection_schedule/source/ics.py
--- a/custom_components/waste_collection_schedule/waste_collection_schedule/source/ics.py
+++ b/custom_components/waste_collection_schedule/waste_collection_schedule/source/ics.py
@@ -13,15 +13,6 @@
 DESCRIPTION = "Source for ICS based schedules."
 URL = None
 TEST_CASES = {
-    "Dortmund, Dudenstr. 5": {
-        "url": "https://www.edg.de/ical/kalender.ics?Strasse=Dudenstr.&Hausnummer=5&Erinnerung=-1&Abfallart=1,2,3,4"
-    },
-    "Leipzig, Sandgrubenweg 27": {
-        "url": "https://stadtreinigung-leipzig.de/wir-kommen-zu-ihnen/abfallkalender/ical.ics?position_nos=38296&name=Sandgrubenweg%2027"
-    },
-    "Ludwigsburg": {
-        "url": "https://kundenportal.avl-lb.de/WasteManagementLudwigsburg/WasteManagementServiceServlet?ApplicationName=Calendar&SubmitAction=sync&StandortID=950230001&AboID=8188&Fra=BT;RT;PT;LT;GT"
-    },
     "Esslingen, Bahnhof": {
         "url": "https://api.abfall.io/?kh=DaA02103019b46345f1998698563DaAd&t=ics&s=1a862df26f6943997cef90233877a4fe"
     },
@@ -39,9 +30,6 @@
         "url": "https://www.awm-muenchen.de/entsorgen/abfuhrkalender?tx_awmabfuhrkalender_abfuhrkalender%5Bhausnummer%5D=11&tx_awmabfuhrkalender_abfuhrkalender%5Bleerungszyklus%5D%5BB%5D=1%2F2%3BU&tx_awmabfuhrkalender_abfuhrkalender%5Bleerungszyklus%5D%5BP%5D=1%2F2%3BG&tx_awmabfuhrkalender_abfuhrkalender%5Bleerungszyklus%5D%5BR%5D=001%3BU&tx_awmabfuhrkalender_abfuhrkalender%5Bsection%5D=ics&tx_awmabfuhrkalender_abfuhrkalender%5Bsinglestandplatz%5D=false&tx_awmabfuhrkalender_abfuhrkalender%5Bstandplatzwahl%5D=true&tx_awmabfuhrkalender_abfuhrkalender%5Bstellplatz%5D%5Bbio%5D=70024507&tx_awmabfuhrkalender_abfuhrkalender%5Bstellplatz%5D%5Bpapier%5D=70024507&tx_awmabfuhrkalender_abfuhrkalender%5Bstellplatz%5D%5Brestmuell%5D=70024507&tx_awmabfuhrkalender_abfuhrkalender%5Bstrasse%5D=bahnstr.&tx_awmabfuhrkalender_abfuhrkalender%5Byear%5D={%Y}",
         "version": 1,
     },
-    "Buxtehude, Am Berg": {
-        "url": "https://abfall.landkreis-stade.de/api_v2/collection_dates/1/ort/10/strasse/90/hausnummern/1/abfallarten/R02-R04-B02-D04-D12-P04-R12-R14-W0-R22-R24-R31/kalender.ics"
-    },
     #    "Hausmüllinfo: ASR Chemnitz": {
     #        "url": "https://asc.hausmuell.info/ics/ics.php",
     #        "method": "POST",
@@ -89,23 +77,10 @@
         },
         "year_field": "year",
     },
-    "Detmold": {
-        "url": "https://abfuhrkalender.detmold.de/icsmaker.php",
-        "method": "GET",
-        "params": {"strid": 338},
-        "year_field": "year",
-    },
-    "EAW Rheingau Taunus": {
-        "url": "https://www.eaw-rheingau-taunus.de/abfallsammlung/abfuhrtermine/feed.ics?tx_vierwdeaw_garbagecalendarics%5Baction%5D=ics&tx_vierwdeaw_garbagecalendarics%5Bcontroller%5D=GarbageCalendar&tx_vierwdeaw_garbagecalendarics%5Bstreet%5D=38",
-        "split_at": ",",
-    },
     "Recollect, Ottawa": {
         "url": "https://recollect.a.ssl.fastly.net/api/places/BCCDF30E-578B-11E4-AD38-5839C200407A/services/208/events.en.ics",
         "split_at": "\\, [and ]*",
     },
-    "Frankfurt am Main, Achenbachstrasse 3": {
-        "url": "https://www.fes-frankfurt.de/abfallkalender/QWNoZW5iYWNoc3RyLnwzfDYwNTk2.ics"
-    },
     "Erlensee, Am Haspel": {
         "url": "https://sperrmuell.erlensee.de/?type=reminder",
         "method": "POST",
diff --git a/update_docu_links.py b/update_docu_links.py
--- a/update_docu_links.py
+++ b/update_docu_links.py
@@ -11,7 +11,7 @@
 SECRET_FILENAME = "secrets.yaml"
 SECRET_REGEX = re.compile(r"!secret\s(\w+)")
 
-BLACK_LIST = {"ics", "static", "example"}
+BLACK_LIST = {"/doc/source/ics.md", "/doc/source/static.md", "/doc/source/example.md"}
 
 START_COUNTRY_SECTION = "<!--Begin of country section-->"
 END_COUNTRY_SECTION = "<!--End of country section-->"
@@ -34,16 +34,47 @@ def end(self):
 
 
 def main():
-    parser = argparse.ArgumentParser(description="Test sources.")
+    parser = argparse.ArgumentParser(description="Update docu links.")
     # args = parser.parse_args()
 
+    sources = []
+
+    sources += browse_sources()
+    sources += browse_ics_yaml()
+
+    # sort into countries
+    country_code_map = make_country_code_map()
+    countries = {}
+
+    orphans = []
+    for s in sources:
+        if s.filename in BLACK_LIST:
+            continue  # skip
+
+        # extract country code
+        code = s.country
+        if code in country_code_map:
+            countries.setdefault(country_code_map[code]["name"], []).append(s)
+        else:
+            orphans.append(s)
+
+    if len(orphans) > 0:
+        print("Orphaned sources without country =========================")
+        for o in orphans:
+            print(o)
+
+    update_readme_md(countries)
+    update_info_md(countries)
+
+
+def browse_sources():
+    """Browse all .py files in the `source` directory"""
     package_dir = (
         Path(__file__).resolve().parents[0]
         / "custom_components"
         / "waste_collection_schedule"
     )
     source_dir = package_dir / "waste_collection_schedule" / "source"
-    print(source_dir)
 
     # add module directory to path
     site.addsitedir(str(package_dir))
@@ -66,9 +97,10 @@ def main():
         url = module.URL
         country = getattr(module, "COUNTRY", f.split("_")[-1])
 
+        filename = f"/doc/source/{f}.md"
         if title is not None:
             sources.append(
-                SourceInfo(filename=f, title=title, url=url, country=country)
+                SourceInfo(filename=filename, title=title, url=url, country=country)
             )
 
         extra_info = getattr(module, "EXTRA_INFO", [])
@@ -77,17 +109,83 @@ def main():
         for e in extra_info:
             sources.append(
                 SourceInfo(
-                    filename=f,
+                    filename=filename,
                     title=e.get("title", title),
                     url=e.get("url", url),
                     country=e.get("country", country),
                 )
             )
 
-    # sort into countries
+    update_awido_de(modules)
+    update_ctrace_de(modules)
+
+    return sources
+
+
+def browse_ics_yaml():
+    """Browse all .yaml files which are descriptions for the ICS source"""
+    doc_dir = Path(__file__).resolve().parents[0] / "doc"
+    yaml_dir = doc_dir / "ics" / "yaml"
+    md_dir = doc_dir / "ics"
+
+    files = yaml_dir.glob("*.yaml")
+    sources = []
+    for f in files:
+        with open(f) as stream:
+            # write markdown file
+            filename = (md_dir / f.name).with_suffix(".md")
+            data = yaml.safe_load(stream)
+            write_ics_md_file(filename, data)
+
+            # extract country code
+            sources.append(
+                SourceInfo(
+                    filename=f"/doc/ics/{filename.name}",
+                    title=data["title"],
+                    url=data["url"],
+                    country=data.get("country", f.stem.split("_")[-1]),
+                )
+            )
+
+    update_ics_md(sources)
+
+    return sources
+
+
+def write_ics_md_file(filename, data):
+    """Write a markdown file for a ICS .yaml file"""
+    md = f"# {data['title']}\n"
+    md += "\n"
+    md += f"{data['title']} is supported by the generic [ICS](/doc/source/ics.md) source. For all available configuration options, please refer to the source description.\n"
+    md += "\n"
+    if "description" in data:
+        md += f"{data['description']}\n"
+    md += "\n"
+    md += "## How to get the configuration arguments\n"
+    md += "\n"
+    md += f"{data['howto']}"
+    md += "\n"
+    md += "## Examples\n"
+    md += "\n"
+    for title, tc in data["test_cases"].items():
+        md += f"### {title}\n"
+        md += "\n"
+        md += "```yaml\n"
+        md += "waste_collection_schedule:\n"
+        md += "  sources:\n"
+        md += "    - name: ics\n"
+        md += "      args:\n"
+        md += multiline_indent(yaml.dump(tc).rstrip("\n"), 8) + "\n"
+        md += "```\n"
+        # md += "\n"
+    with open(filename, "w") as f:
+        f.write(md)
+
+
+def update_ics_md(sources):
     country_code_map = make_country_code_map()
     countries = {}
-    orphans = []
+
     for s in sources:
         if s.filename in BLACK_LIST:
             continue  # skip
@@ -96,18 +194,24 @@ def main():
         code = s.country
         if code in country_code_map:
             countries.setdefault(country_code_map[code]["name"], []).append(s)
-        else:
-            orphans.append(s)
 
-    update_readme_md(countries)
-    update_info_md(countries)
-    update_awido_de(modules)
-    update_ctrace_de(modules)
+    str = ""
+    for country in sorted(countries):
+        str += f"### {country}\n"
+        str += "\n"
 
-    if len(orphans) > 0:
-        print("Orphaned =========================")
-        for o in orphans:
-            print(o)
+        for e in sorted(countries[country], key=lambda e: e.title.lower()):
+            str += f"- [{e.title}]({e.filename}) / {beautify_url(e.url)}\n"
+
+        str += "\n"
+
+    _patch_file("doc/source/ics.md", "service", str)
+
+
+def multiline_indent(s, numspaces):
+    """Indent all lines within the given string by <numspace> spaces"""
+    lines = [(numspaces * " ") + line for line in s.split("\n")]
+    return "\n".join(lines)
 
 
 def beautify_url(url):
@@ -128,9 +232,7 @@ def update_readme_md(countries):
 
         for e in sorted(countries[country], key=lambda e: e.title.lower()):
             # print(f"  {e.title} - {beautify_url(e.url)}")
-            str += (
-                f"- [{e.title}](/doc/source/{e.filename}.md) / {beautify_url(e.url)}\n"
-            )
+            str += f"- [{e.title}]({e.filename}) / {beautify_url(e.url)}\n"
 
         str += "</details>\n"
         str += "\n"


Test patch:
diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/test/test_sources.py b/custom_components/waste_collection_schedule/waste_collection_schedule/test/test_sources.py
--- a/custom_components/waste_collection_schedule/waste_collection_schedule/test/test_sources.py
+++ b/custom_components/waste_collection_schedule/waste_collection_schedule/test/test_sources.py
@@ -33,6 +33,12 @@ def main():
         action="store_true",
         help="Print exception information and stack trace",
     )
+    parser.add_argument(
+        "-I", "--ics", action="store_true", help="Test all .yaml file for ICS source"
+    )
+    parser.add_argument(
+        "-y", "--yaml", action="append", help="Test given .yaml file for ICS source"
+    )
     args = parser.parse_args()
 
     # read secrets.yaml
@@ -53,15 +59,21 @@ def main():
     # add module directory to path
     site.addsitedir(str(package_dir))
 
+    # find all source files for testing
     if args.source is not None:
-        files = args.source
-    else:
-        files = filter(
+        # source file(s) given
+        source_files = args.source
+    elif not args.ics and args.yaml is None:
+        # no ICS yaml files given --> test all source files
+        source_files = filter(
             lambda x: x != "__init__",
             map(lambda x: x.stem, source_dir.glob("*.py")),
         )
+    else:
+        # ICS yaml file(s) given
+        source_files = []
 
-    for f in sorted(files):
+    for f in sorted(source_files):
         # iterate through all *.py files in waste_collection_schedule/source
         print(f"Testing source {f} ...")
         module = importlib.import_module(f"waste_collection_schedule.source.{f}")
@@ -80,49 +92,65 @@ def main():
             # replace secrets in arguments
             replace_secret(secrets, tc)
 
-            # create source
-            try:
-                source = module.Source(**tc)
-                result = source.fetch()
-                count = len(result)
-                if count > 0:
-                    print(
-                        f"  found {bcolors.OKGREEN}{count}{bcolors.ENDC} entries for {name}"
-                    )
-                else:
-                    print(
-                        f"  found {bcolors.WARNING}0{bcolors.ENDC} entries for {name}"
-                    )
-
-                # test if source is returning the correct date format
-                if (
-                    len(
-                        list(
-                            filter(lambda x: type(x.date) is not datetime.date, result)
-                        )
-                    )
-                    > 0
-                ):
-                    print(
-                        f"{bcolors.FAIL}  ERROR: source returns invalid date format (datetime.datetime instead of datetime.date?){bcolors.ENDC}"
-                    )
-
-                if args.list:
-                    result = (
-                        sorted(result, key=lambda x: x.date) if args.sorted else result
-                    )
-                    for x in result:
-                        icon_str = f" [{x.icon}]" if args.icon else ""
-                        weekday_str = x.date.strftime("%a ") if args.weekday else ""
-                        print(
-                            f"    {x.date.isoformat()} {weekday_str}: {x.type}{icon_str}"
-                        )
-            except KeyboardInterrupt:
-                exit()
-            except Exception as exc:
-                print(f"  {name} {bcolors.FAIL}failed{bcolors.ENDC}: {exc}")
-                if args.traceback:
-                    print(indent(traceback.format_exc(), 4))
+            test_fetch(module, name, tc, args)
+
+    # find all ICS yaml files for testing
+    ics_yaml_dir = Path(__file__).resolve().parents[4] / "doc" / "ics" / "yaml"
+    if args.ics:
+        # test all ICS yaml files
+        yaml_files = ics_yaml_dir.glob("*.yaml")
+    elif args.yaml:
+        # ICS yaml files specified
+        yaml_files = [Path(ics_yaml_dir, f).with_suffix(".yaml") for f in args.yaml]
+    elif args.source is None:
+        # neither source nor ICS yaml files specified --> test all yaml files
+        yaml_files = ics_yaml_dir.glob("*.yaml")
+    else:
+        # source files given --> don't test ICS yaml files
+        yaml_files = []
+
+    # run through all .yaml files for ICS source
+    module = importlib.import_module("waste_collection_schedule.source.ics")
+    for f in sorted(yaml_files):
+        print(f"Testing ICS {f.stem}")
+        with open(f) as stream:
+            # read yaml file
+            data = yaml.safe_load(stream)
+
+            # run through all test-cases
+            for name, tc in data["test_cases"].items():
+                test_fetch(module, name, tc, args)
+
+
+def test_fetch(module, name, tc, args):
+    # create source
+    try:
+        source = module.Source(**tc)
+        result = source.fetch()
+        count = len(result)
+        if count > 0:
+            print(f"  found {bcolors.OKGREEN}{count}{bcolors.ENDC} entries for {name}")
+        else:
+            print(f"  found {bcolors.WARNING}0{bcolors.ENDC} entries for {name}")
+
+        # test if source is returning the correct date format
+        if len(list(filter(lambda x: type(x.date) is not datetime.date, result))) > 0:
+            print(
+                f"{bcolors.FAIL}  ERROR: source returns invalid date format (datetime.datetime instead of datetime.date?){bcolors.ENDC}"
+            )
+
+        if args.list:
+            result = sorted(result, key=lambda x: x.date) if args.sorted else result
+            for x in result:
+                icon_str = f" [{x.icon}]" if args.icon else ""
+                weekday_str = x.date.strftime("%a ") if args.weekday else ""
+                print(f"    {x.date.isoformat()} {weekday_str}: {x.type}{icon_str}")
+    except KeyboardInterrupt:
+        exit()
+    except Exception as exc:
+        print(f"  {name} {bcolors.FAIL}failed{bcolors.ENDC}: {exc}")
+        if args.traceback:
+            print(indent(traceback.format_exc(), 4))
 
 
 def replace_secret(secrets, d):