Repository: mampfes/hacs_waste_collection_schedule

Issue:
[Bug]: No data anymore for Gateshead Council
### I Have A Problem With:

A specific source

### What's Your Problem

All of a sudden I am not getting any data through anymore for Gateshead council.

### Source (if relevant)

gateshead_gov_uk

### Logs

```Shell
2024-04-25 13:31:32.481 ERROR (SyncWorker_8) [waste_collection_schedule.source_shell] fetch failed for source Gateshead Council:
File "/config/custom_components/waste_collection_schedule/waste_collection_schedule/source_shell.py", line 134, in fetch
File "/config/custom_components/waste_collection_schedule/waste_collection_schedule/source/gateshead_gov_uk.py", line 44, in fetch
```


### Relevant Configuration

_No response_

### Checklist Source Error

- [X] Use the example parameters for your source (often available in the documentation) (don't forget to restart Home Assistant after changing the configuration)
- [X] Checked that the website of your service provider is still working
- [ ] Tested my attributes on the service provider website (if possible)
- [X] I have tested with the latest version of the integration (master) (for HACS in the 3 dot menu of the integration click on "Redownload" and choose master as version)

### Checklist Sensor Error

- [X] Checked in the Home Assistant Calendar tab if the event names match the types names (if types argument is used)

### Required

- [X] I have searched past (closed AND opened) issues to see if this bug has already been reported, and it hasn't been.
- [X] I understand that people give their precious time for free, and thus I've done my very best to make this problem as easy as possible to investigate.


Discussion:

Resolving patch:
diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/source/gateshead_gov_uk.py b/custom_components/waste_collection_schedule/waste_collection_schedule/source/gateshead_gov_uk.py
--- a/custom_components/waste_collection_schedule/waste_collection_schedule/source/gateshead_gov_uk.py
+++ b/custom_components/waste_collection_schedule/waste_collection_schedule/source/gateshead_gov_uk.py
@@ -1,11 +1,11 @@
-from datetime import datetime, timedelta
+import base64
+import json
+import re
+from datetime import datetime
 
 import requests
-import re
-import json
-import base64
 from bs4 import BeautifulSoup
-from waste_collection_schedule import Collection # type: ignore[attr-defined]
+from waste_collection_schedule import Collection  # type: ignore[attr-defined]
 
 TITLE = "Gateshead Council"
 DESCRIPTION = "Source for gateshead.gov.uk services for Gateshead"
@@ -13,7 +13,8 @@
 
 TEST_CASES = {
     "Test_001": {"uprn": "100000077407"},
-    "Test_002": {"uprn": "100000058404"},
+    "Test_002": {"uprn": 100000058404},
+    "Test_003": {"uprn": 100000033887},
 }
 
 ICON_MAP = {
@@ -29,37 +30,48 @@ def __init__(self, uprn):
 
     def fetch(self):
         session = requests.Session()
-        
+
         # Start a session
-        r = session.get("https://www.gateshead.gov.uk/article/3150/Bin-collection-day-checker")
-        
+        r = session.get(
+            "https://www.gateshead.gov.uk/article/3150/Bin-collection-day-checker"
+        )
+
         r.raise_for_status()
         soup = BeautifulSoup(r.text, features="html.parser")
 
         # Extract form submission url and form data
-        form_url = soup.find("form", attrs={"id": "BINCOLLECTIONCHECKER_FORM"})["action"]
-        pageSessionId = soup.find("input", attrs={"name": "BINCOLLECTIONCHECKER_PAGESESSIONID"})["value"]
-        sessionId = soup.find("input", attrs={"name": "BINCOLLECTIONCHECKER_SESSIONID"})["value"]
-        nonce = soup.find("input", attrs={"name": "BINCOLLECTIONCHECKER_NONCE"})["value"]
-        ticks = soup.find("input", attrs={"name": "BINCOLLECTIONCHECKER_ADDRESSSEARCH_TICKS"})["value"]
-        
+        form_url = soup.find("form", attrs={"id": "BINCOLLECTIONCHECKER_FORM"})[
+            "action"
+        ]
+        pageSessionId = soup.find(
+            "input", attrs={"name": "BINCOLLECTIONCHECKER_PAGESESSIONID"}
+        )["value"]
+        sessionId = soup.find(
+            "input", attrs={"name": "BINCOLLECTIONCHECKER_SESSIONID"}
+        )["value"]
+        nonce = soup.find("input", attrs={"name": "BINCOLLECTIONCHECKER_NONCE"})[
+            "value"
+        ]
+
         form_data = {
             "BINCOLLECTIONCHECKER_PAGESESSIONID": pageSessionId,
             "BINCOLLECTIONCHECKER_SESSIONID": sessionId,
             "BINCOLLECTIONCHECKER_NONCE": nonce,
-            "BINCOLLECTIONCHECKER_ADDRESSSEARCH_TICKS": ticks,
+            # "BINCOLLECTIONCHECKER_ADDRESSSEARCH_TICKS": ticks,
             "BINCOLLECTIONCHECKER_FORMACTION_NEXT": "BINCOLLECTIONCHECKER_ADDRESSSEARCH_NEXTBUTTON",
             "BINCOLLECTIONCHECKER_ADDRESSSEARCH_UPRN": self._uprn,
-            "BINCOLLECTIONCHECKER_ADDRESSSEARCH_ADDRESSTEXT": " " # Not quite sure why this is need (can not be empty) maybe used if there are multiple matches=??? But UPRN should be unique???
+            "BINCOLLECTIONCHECKER_ADDRESSSEARCH_ADDRESSTEXT": " ",  # Not quite sure why this is need (can not be empty) maybe used if there are multiple matches=??? But UPRN should be unique???
         }
 
         # Submit form
         r = session.post(form_url, data=form_data)
         r.raise_for_status()
-        
+
         # Extract encoded response data
         soup = BeautifulSoup(r.text, features="html.parser")
-        pattern = re.compile(r"var BINCOLLECTIONCHECKERFormData = \"(.*?)\";$", re.MULTILINE | re.DOTALL)
+        pattern = re.compile(
+            r"var BINCOLLECTIONCHECKERFormData = \"(.*?)\";$", re.MULTILINE | re.DOTALL
+        )
         script = soup.find("script", text=pattern)
 
         response_data = pattern.search(script.text).group(1)
@@ -67,23 +79,43 @@ def fetch(self):
         # Decode base64 encoded response data and convert to JSON
         decoded_data = base64.b64decode(response_data)
         data = json.loads(decoded_data)
-        
+        soup = BeautifulSoup(
+            data["HOUSEHOLDCOLLECTIONS_1"]["DISPLAYHOUSEHOLD"], features="html.parser"
+        )
+
         # Extract entries
         entries = []
-        for bin in data["HOUSEHOLDCOLLECTIONS_1"]["DISPLAYHOUSEHOLD2"]["propertyCollections"]["future"]:
-            dt = datetime.strptime(bin['collectionDate'], '%Y-%m-%dT%H:%M:%S.%fZ')
-            if dt.hour == 23: # Some collections are returned as 11pm night before the collection
-                dt += timedelta(hours=1)
-            date = dt.date()
-            types = bin['wasteTypeCode'].split("|")
+        month = None
+        for tr in soup.find_all("tr"):
+            month_th = tr.find("th", attrs={"colspan": "3"})
+            if month_th:
+                month = month_th.text.split(" ")[
+                    0
+                ]  # split if month is followed by year (may happen in December, not sure)
+                continue
+            if not month:
+                continue
+
+            tds = tr.find_all("td")
+            if len(tds) != 3:
+                continue
+            day = tds[0].text
+            types = tds[2].text.split(" and ")
+
+            now = datetime.now()
+            dt = datetime.strptime(f"{now.year}-{month}-{day}", "%Y-%B-%d").date()
+            if dt.month in (1, 2, 3) and now.month in (
+                11,
+                12,
+            ):  # fix Dates for next year
+                dt = dt.replace(year=now.year + 1)
             for type in types:
                 entries.append(
                     Collection(
-                        date=date,
-                        t=type,
+                        date=dt,
+                        t=type.strip(),
                         icon=ICON_MAP.get(type),
                     )
                 )
-                    
 
         return entries


Test patch: