Repository: alexdlaird/amazon-orders

Issue:
BeautifulSoup html.parser not correctly parsing some item titles
### Acknowledgements

- [x] I have written a descriptive issue title
- [x] I have searched [Issues](issues?q=is%3Aissue) to see if the bug has already been reported
- [x] I have searched [Stack Overflow](https://stackoverflow.com/questions/) to ensure the issue I'm experiencing has not already been discussed
- [x] If possible, I have [enabled `debug` mode](https://amazon-orders.readthedocs.io/troubleshooting.html#enable-debug-mode) and am attaching relevant console logs and HTML files

### Operating System

Debian Linux 12.8

### Python Version

3.11.2

### amazon-orders Version

3.2.2

### Describe the Bug

When parsing the following item string with the current BeautifulSoup configured parser: `html.parser`, the `&` at the beginning of the resulting text is mangled.  Changing the parser to `lxml` fixed the issue.

```python
>>> from bs4 import BeautifulSoup
>>> from pathlib import Path
>>> html = open(Path("~/Downloads/Order Details.html").expanduser()).read()
>>> soup = BeautifulSoup(html, 'html.parser')
>>> soup.select("[data-component='itemTitle']")[0].text.strip()
'⩓ Per Se Lined Journal Notebook, 160 Pages, A5 5.7 x 8 inches College Ruled Thick Paper Notebook Journals for Writing, Hardcover Notebooks for Women Men Office School (1 Pack, Royal Blue)'
>>> # Mangled above
>>> soup = BeautifulSoup(html, 'lxml')
>>> soup.select("[data-component='itemTitle']")[0].text.strip()
'&And Per Se Lined Journal Notebook, 160 Pages, A5 5.7 x 8 inches College Ruled Thick Paper Notebook Journals for Writing, Hardcover Notebooks for Women Men Office School (1 Pack, Royal Blue)'
```

Here's a snippet of the html with the problem item title: <https://fattuba.com/download/problem_item_title.html>

Here's a link to the product page: <https://www.amazon.com/dp/B0CW5Y6PKG>

### Steps to Reproduce

1. Run `amazon_orders.get_order_history(year=this_year, full_details=True)` with an account with that particular item in an order.
2. See the resulting item.title is mangled.

### Expected Behavior

_No response_

Discussion:
@alexdlaird: This is now configurable in release `3.2.3`. Thanks for contributing!@tubaman: More info for posterity: `html.parser` works fine for the same item title under shipments. I believe this is because in one case the ampersand is escaped as an HTML entity and not in the other case.

Solution patch:
diff --git a/amazonorders/conf.py b/amazonorders/conf.py
index 830a0e5..d614406 100644
--- a/amazonorders/conf.py
+++ b/amazonorders/conf.py
@@ -36,6 +36,7 @@ class AmazonOrdersConfig:
             "order_class": "amazonorders.entity.order.Order",
             "shipment_class": "amazonorders.entity.shipment.Shipment",
             "item_class": "amazonorders.entity.item.Item",
+            "bs4_parser": "html.parser",
         }
 
         if os.path.exists(self.config_path):
diff --git a/amazonorders/entity/order.py b/amazonorders/entity/order.py
index c98fe93..f30fd93 100644
--- a/amazonorders/entity/order.py
+++ b/amazonorders/entity/order.py
@@ -146,7 +146,7 @@ class Order(Parsable):
                 data_popover = value.get("data-a-popover", {})  # type: ignore[arg-type, var-annotated]
                 inline_content = data_popover.get("inlineContent")  # type: ignore[union-attr]
                 if inline_content:
-                    value = BeautifulSoup(json.loads(inline_content), "html.parser")
+                    value = BeautifulSoup(json.loads(inline_content), self.config.bs4_parser)
 
         if not value:
             # TODO: there are multiple shipToData tags, we should double check we're picking the right one
@@ -166,7 +166,7 @@ class Order(Parsable):
             )
 
             if parent_tag:
-                value = BeautifulSoup(str(parent_tag.contents[0]).strip(), "html.parser")
+                value = BeautifulSoup(str(parent_tag.contents[0]).strip(), self.config.bs4_parser)
 
         if not value:
             return None
diff --git a/amazonorders/session.py b/amazonorders/session.py
index ddbe41e..53a8437 100644
--- a/amazonorders/session.py
+++ b/amazonorders/session.py
@@ -141,7 +141,7 @@ class AmazonSession:
 
         self.last_response = self.session.request(method, url, **kwargs)
         self.last_response_parsed = BeautifulSoup(self.last_response.text,
-                                                  "html.parser")
+                                                  self.config.bs4_parser)
 
         cookies = dict_from_cookiejar(self.session.cookies)
         if os.path.exists(self.config.cookie_jar_path):
diff --git a/scripts/build-test-resources.py b/scripts/build-test-resources.py
index e7e4487..a81682e 100755
--- a/scripts/build-test-resources.py
+++ b/scripts/build-test-resources.py
@@ -22,7 +22,7 @@ def _obfuscate(response_parsed, hide_data_rules):
     for rule in hide_data_rules:
         pattern = re.compile(rule["find_pattern"], re.IGNORECASE)
         cleaned = pattern.sub(rule["replace"], cleaned)
-    return BeautifulSoup(cleaned, "html.parser")
+    return BeautifulSoup(cleaned, self.config.bs4_parser)
 
 
 def build_test_resources(args):
@@ -85,7 +85,7 @@ def build_test_resources(args):
         if page["type"] == "order-details":
             url = f"{config.constants.ORDER_DETAILS_URL}?orderID={page['order-id']}"
             response = amazon_session.get(url)
-            response_parsed = BeautifulSoup(response.text, "html.parser")
+            response_parsed = BeautifulSoup(response.text, self.config.bs4_parser)
 
             cleaned_response = _obfuscate(response_parsed, hide_data_rules)
 
@@ -93,7 +93,7 @@ def build_test_resources(args):
         else:
             url = f"{config.constants.ORDER_HISTORY_URL}?timeFilter=year-{page['year']}&startIndex={page['start-index']}"
             response = amazon_session.get(url)
-            response_parsed = BeautifulSoup(response.text, "html.parser")
+            response_parsed = BeautifulSoup(response.text, self.config.bs4_parser)
 
             cleaned_response = _obfuscate(response_parsed, hide_data_rules)
 


Test patch:
diff --git a/tests/test_conf.py b/tests/test_conf.py
index 4cef259..a8436f5 100644
--- a/tests/test_conf.py
+++ b/tests/test_conf.py
@@ -48,7 +48,8 @@ class TestConf(TestCase):
         # THEN
         self.assertTrue(os.path.exists(config_path))
         with open(config.config_path, "r") as f:
-            self.assertEqual("""constants_class: amazonorders.constants.Constants
+            self.assertEqual("""bs4_parser: html.parser
+constants_class: amazonorders.constants.Constants
 cookie_jar_path: {}
 item_class: amazonorders.entity.item.Item
 max_auth_attempts: 10