CVE-2024-0243: Server-Side Request Forgery (SSRF) (CWE-918)
value: With the following crawler configuration:

```python
from bs4 import BeautifulSoup as Soup

url = "https://example.com"
loader = RecursiveUrlLoader(
    url=url, max_depth=2, extractor=lambda x: Soup(x, "html.parser").text
)
docs = loader.load()
```

An attacker in control of the contents of `https://example.com` could place a malicious HTML file in there with links like "https://example.completely.different/my_file.html" and the crawler would proceed to download that file as well even though `prevent_outside=True`.

https://github.com/langchain-ai/langchain/blob/bf0b3cc0b5ade1fb95a5b1b6fa260e99064c2e22/libs/community/langchain_community/document_loaders/recursive_url_loader.py#L51-L51

Resolved in https://github.com/langchain-ai/langchain/pull/15559
CWE description: The web server receives a URL or similar request from an upstream component and retrieves the contents of this URL, but it does not sufficiently ensure that the request is being sent to the expected destination.
Published: 2024-02-26T16:27Z
Repository: https://github.com/langchain-ai/langchain
Language: Python
Files changed: libs/core/langchain_core/utils/html.py, libs/core/tests/unit_tests/utils/test_html.py
Commit date: 2024-01-04 16:33:57 -0500
Commit message: core[patch]: Further restrict recursive URL loader (#15559)

Includes code from this PR:  https://github.com/langchain-ai/langchain/compare/HEAD...m0kr4n3:security/fix_ssrf 
with additional fixes 

Unit tests cover new test cases
Diff: diff --git a/libs/core/langchain_core/utils/html.py b/libs/core/langchain_core/utils/html.py
index 95e1c3c2f..bbea15f0f 100644
--- a/libs/core/langchain_core/utils/html.py
+++ b/libs/core/langchain_core/utils/html.py
@@ -65,25 +65,39 @@ def extract_sub_links(
         exclude_prefixes: Exclude any URLs that start with one of these prefixes.
 
     Returns:
         List[str]: sub links
     """
-    base_url = base_url if base_url is not None else url
+    base_url_to_use = base_url if base_url is not None else url
+    parsed_base_url = urlparse(base_url_to_use)
     all_links = find_all_links(raw_html, pattern=pattern)
     absolute_paths = set()
     for link in all_links:
+        parsed_link = urlparse(link)
         # Some may be absolute links like https://to/path
-        if link.startswith("http"):
-            absolute_paths.add(link)
+        if parsed_link.scheme == "http" or parsed_link.scheme == "https":
+            absolute_path = link
         # Some may have omitted the protocol like //to/path
         elif link.startswith("//"):
-            absolute_paths.add(f"{urlparse(url).scheme}:{link}")
+            absolute_path = f"{urlparse(url).scheme}:{link}"
         else:
-            absolute_paths.add(urljoin(url, link))
-    res = []
+            absolute_path = urljoin(url, parsed_link.path)
+        absolute_paths.add(absolute_path)
+
+    results = []
     for path in absolute_paths:
-        if any(path.startswith(exclude) for exclude in exclude_prefixes):
-            continue
-        if prevent_outside and not path.startswith(base_url):
+        if any(path.startswith(exclude_prefix) for exclude_prefix in exclude_prefixes):
             continue
-        res.append(path)
-    return res
+
+        if prevent_outside:
+            parsed_path = urlparse(path)
+
+            if parsed_base_url.netloc != parsed_path.netloc:
+                continue
+
+            # Will take care of verifying rest of path after netloc
+            # if it's more specific
+            if not path.startswith(base_url_to_use):
+                continue
+
+        results.append(path)
+    return results
diff --git a/libs/core/tests/unit_tests/utils/test_html.py b/libs/core/tests/unit_tests/utils/test_html.py
index 117d3698a..a2c80f6e6 100644
--- a/libs/core/tests/unit_tests/utils/test_html.py
+++ b/libs/core/tests/unit_tests/utils/test_html.py
@@ -154,5 +154,32 @@ def test_extract_sub_links_exclude() -> None:
             prevent_outside=False,
             exclude_prefixes=("https://foobar.com/how", "http://baz.org"),
         )
     )
     assert actual == expected
+
+
+def test_prevent_outside() -> None:
+    """Test that prevent outside compares against full base URL."""
+    html = (
+        '<a href="https://foobar.comic.com">BAD</a>'
+        '<a href="https://foobar.comic:9999">BAD</a>'
+        '<a href="https://foobar.com:9999">BAD</a>'
+        '<a href="http://foobar.com:9999/">BAD</a>'
+        '<a href="https://foobar.com/OK">OK</a>'
+        '<a href="http://foobar.com/BAD">BAD</a>'  # Change in scheme is not OK here
+    )
+
+    expected = sorted(
+        [
+            "https://foobar.com/OK",
+        ]
+    )
+    actual = sorted(
+        extract_sub_links(
+            html,
+            "https://foobar.com/hello/bill.html",
+            base_url="https://foobar.com",
+            prevent_outside=True,
+        )
+    )
+    assert actual == expected

Vulnerable code:     base_url = base_url if base_url is not None else url
        if link.startswith("http"):
            absolute_paths.add(link)
            absolute_paths.add(f"{urlparse(url).scheme}:{link}")
            absolute_paths.add(urljoin(url, link))
    res = []
        if any(path.startswith(exclude) for exclude in exclude_prefixes):
            continue
        if prevent_outside and not path.startswith(base_url):
        res.append(path)
    return res
Fixed code:     base_url_to_use = base_url if base_url is not None else url
    parsed_base_url = urlparse(base_url_to_use)
        parsed_link = urlparse(link)
        if parsed_link.scheme == "http" or parsed_link.scheme == "https":
            absolute_path = link
            absolute_path = f"{urlparse(url).scheme}:{link}"
            absolute_path = urljoin(url, parsed_link.path)
        absolute_paths.add(absolute_path)

    results = []
        if any(path.startswith(exclude_prefix) for exclude_prefix in exclude_prefixes):

        if prevent_outside:
            parsed_path = urlparse(path)

            if parsed_base_url.netloc != parsed_path.netloc:
                continue

            # Will take care of verifying rest of path after netloc
            # if it's more specific
            if not path.startswith(base_url_to_use):
                continue

        results.append(path)
    return results


def test_prevent_outside() -> None:
    """Test that prevent outside compares against full base URL."""
    html = (
        '<a href="https://foobar.comic.com">BAD</a>'
        '<a href="https://foobar.comic:9999">BAD</a>'
        '<a href="https://foobar.com:9999">BAD</a>'
        '<a href="http://foobar.com:9999/">BAD</a>'
        '<a href="https://foobar.com/OK">OK</a>'
        '<a href="http://foobar.com/BAD">BAD</a>'  # Change in scheme is not OK here
    )

    expected = sorted(
        [
            "https://foobar.com/OK",
        ]
    )
    actual = sorted(
        extract_sub_links(
            html,
            "https://foobar.com/hello/bill.html",
            base_url="https://foobar.com",
            prevent_outside=True,
        )
    )
    assert actual == expected