CVE-2023-34457 (Python)

MechanicalSoup is a Python library for automating interaction with websites. Starting in version 0.2.0 and prior to version 1.3.0, a malicious web server can read arbitrary files on the client using a `<input type="file" ...>` inside HTML form. All users of MechanicalSoup's form submission are affected, unless they took very specific (and manual) steps to reset HTML form field values. Version 1.3.0 contains a patch for this issue.

Weakness:
CWE-20: name: Improper Input Validation
description: The product receives input or data, but it does
        not validate or incorrectly validates that the input has the
        properties that are required to process the data safely and
        correctly.

Repository: https://github.com/MechanicalSoup/MechanicalSoup

Vulnerable code:
file_path: mechanicalsoup/browser.py
snippet:     @classmethod
    def get_request_kwargs(cls, form, url=None, **kwargs):
        """Extract input data from the form."""
        method = str(form.get("method", "get"))
        action = form.get("action")
        url = urllib.parse.urljoin(url, action)
        if url is None:  # This happens when both `action` and `url` are None.
            raise ValueError('no URL to submit to')

        # read https://www.w3.org/TR/html52/sec-forms.html
        if method.lower() == "get":
            data = kwargs.pop("params", dict())
        else:
            data = kwargs.pop("data", dict())
        files = kwargs.pop("files", dict())

        # Use a list of 2-tuples to better reflect the behavior of browser QSL.
        # Requests also retains order when encoding form data in 2-tuple lists.
        data = [(k, v) for k, v in data.items()]

        multipart = form.get("enctype", "") == "multipart/form-data"

        # Process form tags in the order that they appear on the page,
        # skipping those tags that do not have a name-attribute.
        selector = ",".join(f"{tag}[name]" for tag in
                            ("input", "button", "textarea", "select"))
        for tag in form.select(selector):
            name = tag.get("name")  # name-attribute of tag

            # Skip disabled elements, since they should not be submitted.
            if tag.has_attr('disabled'):
                continue

            if tag.name == "input":
                if tag.get("type", "").lower() in ("radio", "checkbox"):
                    if "checked" not in tag.attrs:
                        continue
                    value = tag.get("value", "on")
                else:
                    # browsers use empty string for inputs with missing values
                    value = tag.get("value", "")

                # If the enctype is not multipart, the filename is put in
                # the form as a text input and the file is not sent.
                if tag.get("type", "").lower() == "file" and multipart:
                    filepath = value
                    if filepath != "" and isinstance(filepath, str):
                        content = open(filepath, "rb")
                    else:
                        content = ""
                    filename = os.path.basename(filepath)
                    # If value is the empty string, we still pass it
                    # for consistency with browsers (see
                    # https://github.com/MechanicalSoup/MechanicalSoup/issues/250).
                    files[name] = (filename, content)
                else:
                    data.append((name, value))

            elif tag.name == "button":
                if tag.get("type", "").lower() in ("button", "reset"):
                    continue
                else:
                    data.append((name, tag.get("value", "")))

            elif tag.name == "textarea":
                data.append((name, tag.text))

            elif tag.name == "select":
                # If the value attribute is not specified, the content will
                # be passed as a value instead.
                options = tag.select("option")
                selected_values = [i.get("value", i.text) for i in options
                                   if "selected" in i.attrs]
                if "multiple" in tag.attrs:
                    for value in selected_values:
                        data.append((name, value))
                elif selected_values:
                    # A standard select element only allows one option to be
                    # selected, but browsers pick last if somehow multiple.
                    data.append((name, selected_values[-1]))
                elif options:
                    # Selects the first option if none are selected
                    first_value = options[0].get("value", options[0].text)
                    data.append((name, first_value))

        if method.lower() == "get":
            kwargs["params"] = data
        else:
            kwargs["data"] = data

        # The following part of the function is here to respect the
        # enctype specified by the form, i.e. force sending multipart
        # content. Since Requests doesn't have yet a feature to choose
        # enctype, we have to use tricks to make it behave as we want
        # This code will be updated if Requests implements it.
        if multipart and not files:
            # Requests will switch to "multipart/form-data" only if
            # files pass the `if files:` test, so in this case we use
            # a modified dict that passes the if test even if empty.
            class DictThatReturnsTrue(dict):
                def __bool__(self):
                    return True
                __nonzero__ = __bool__

            files = DictThatReturnsTrue()

        return cls._get_request_kwargs(method, url, files=files, **kwargs)

file_path: mechanicalsoup/form.py
snippet:     def set(self, name, value, force=False):
        """Set a form element identified by ``name`` to a specified ``value``.
        The type of element (input, textarea, select, ...) does not
        need to be given; it is inferred by the following methods:
        :func:`~Form.set_checkbox`,
        :func:`~Form.set_radio`,
        :func:`~Form.set_input`,
        :func:`~Form.set_textarea`,
        :func:`~Form.set_select`.
        If none of these methods find a matching element, then if ``force``
        is True, a new element (``<input type="text" ...>``) will be
        added using :func:`~Form.new_control`.

        Example: filling-in a login/password form with EULA checkbox

        .. code-block:: python

            form.set("login", username)
            form.set("password", password)
            form.set("eula-checkbox", True)

        Example: uploading a file through a ``<input type="file"
        name="tagname">`` field (provide the path to the local file,
        and its content will be uploaded):

        .. code-block:: python

            form.set("tagname", path_to_local_file)

        """
        for func in ("checkbox", "radio", "input", "textarea", "select"):
            try:
                getattr(self, "set_" + func)({name: value})
                return
            except InvalidFormMethod:
                pass
        if force:
            self.new_control('text', name, value=value)
            return
        raise LinkNotFoundError("No valid element named " + name)

file_path: mechanicalsoup/form.py
snippet:     def new_control(self, type, name, value, **kwargs):
        """Add a new input element to the form.

        The arguments set the attributes of the new element.
        """
        # Remove existing input-like elements with the same name
        for tag in ('input', 'textarea', 'select'):
            for old in self.form.find_all(tag, {'name': name}):
                old.decompose()
        # We don't have access to the original soup object (just the
        # Tag), so we instantiate a new BeautifulSoup() to call
        # new_tag(). We're only building the soup object, not parsing
        # anything, so the parser doesn't matter. Specify the one
        # included in Python to avoid having dependency issue.
        control = BeautifulSoup("", "html.parser").new_tag('input')
        control['type'] = type
        control['name'] = name
        control['value'] = value
        for k, v in kwargs.items():
            control[k] = v
        self.form.append(control)
        return control

Fixed code:
file_path: mechanicalsoup/browser.py
snippet:     @classmethod
    def get_request_kwargs(cls, form, url=None, **kwargs):
        """Extract input data from the form."""
        method = str(form.get("method", "get"))
        action = form.get("action")
        url = urllib.parse.urljoin(url, action)
        if url is None:  # This happens when both `action` and `url` are None.
            raise ValueError('no URL to submit to')

        # read https://www.w3.org/TR/html52/sec-forms.html
        if method.lower() == "get":
            data = kwargs.pop("params", dict())
        else:
            data = kwargs.pop("data", dict())
        files = kwargs.pop("files", dict())

        # Use a list of 2-tuples to better reflect the behavior of browser QSL.
        # Requests also retains order when encoding form data in 2-tuple lists.
        data = [(k, v) for k, v in data.items()]

        multipart = form.get("enctype", "") == "multipart/form-data"

        # Process form tags in the order that they appear on the page,
        # skipping those tags that do not have a name-attribute.
        selector = ",".join(f"{tag}[name]" for tag in
                            ("input", "button", "textarea", "select"))
        for tag in form.select(selector):
            name = tag.get("name")  # name-attribute of tag

            # Skip disabled elements, since they should not be submitted.
            if tag.has_attr('disabled'):
                continue

            if tag.name == "input":
                if tag.get("type", "").lower() in ("radio", "checkbox"):
                    if "checked" not in tag.attrs:
                        continue
                    value = tag.get("value", "on")
                else:
                    # browsers use empty string for inputs with missing values
                    value = tag.get("value", "")

                # If the enctype is not multipart, the filename is put in
                # the form as a text input and the file is not sent.
                if is_multipart_file_upload(form, tag):
                    if isinstance(value, io.IOBase):
                        content = value
                        filename = os.path.basename(getattr(value, "name", ""))
                    else:
                        content = ""
                        filename = os.path.basename(value)
                    # If content is the empty string, we still pass it
                    # for consistency with browsers (see
                    # https://github.com/MechanicalSoup/MechanicalSoup/issues/250).
                    files[name] = (filename, content)
                else:
                    if isinstance(value, io.IOBase):
                        value = os.path.basename(getattr(value, "name", ""))
                    data.append((name, value))

            elif tag.name == "button":
                if tag.get("type", "").lower() in ("button", "reset"):
                    continue
                else:
                    data.append((name, tag.get("value", "")))

            elif tag.name == "textarea":
                data.append((name, tag.text))

            elif tag.name == "select":
                # If the value attribute is not specified, the content will
                # be passed as a value instead.
                options = tag.select("option")
                selected_values = [i.get("value", i.text) for i in options
                                   if "selected" in i.attrs]
                if "multiple" in tag.attrs:
                    for value in selected_values:
                        data.append((name, value))
                elif selected_values:
                    # A standard select element only allows one option to be
                    # selected, but browsers pick last if somehow multiple.
                    data.append((name, selected_values[-1]))
                elif options:
                    # Selects the first option if none are selected
                    first_value = options[0].get("value", options[0].text)
                    data.append((name, first_value))

        if method.lower() == "get":
            kwargs["params"] = data
        else:
            kwargs["data"] = data

        # The following part of the function is here to respect the
        # enctype specified by the form, i.e. force sending multipart
        # content. Since Requests doesn't have yet a feature to choose
        # enctype, we have to use tricks to make it behave as we want
        # This code will be updated if Requests implements it.
        if multipart and not files:
            # Requests will switch to "multipart/form-data" only if
            # files pass the `if files:` test, so in this case we use
            # a modified dict that passes the if test even if empty.
            class DictThatReturnsTrue(dict):
                def __bool__(self):
                    return True
                __nonzero__ = __bool__

            files = DictThatReturnsTrue()

        return cls._get_request_kwargs(method, url, files=files, **kwargs)

file_path: mechanicalsoup/form.py
snippet:     def set(self, name, value, force=False):
        """Set a form element identified by ``name`` to a specified ``value``.
        The type of element (input, textarea, select, ...) does not
        need to be given; it is inferred by the following methods:
        :func:`~Form.set_checkbox`,
        :func:`~Form.set_radio`,
        :func:`~Form.set_input`,
        :func:`~Form.set_textarea`,
        :func:`~Form.set_select`.
        If none of these methods find a matching element, then if ``force``
        is True, a new element (``<input type="text" ...>``) will be
        added using :func:`~Form.new_control`.

        Example: filling-in a login/password form with EULA checkbox

        .. code-block:: python

            form.set("login", username)
            form.set("password", password)
            form.set("eula-checkbox", True)

        Example: uploading a file through a ``<input type="file"
        name="tagname">`` field (provide an open file object,
        and its content will be uploaded):

        .. code-block:: python

            form.set("tagname", open(path_to_local_file, "rb"))

        """
        for func in ("checkbox", "radio", "input", "textarea", "select"):
            try:
                getattr(self, "set_" + func)({name: value})
                return
            except InvalidFormMethod:
                pass
        if force:
            self.new_control('text', name, value=value)
            return
        raise LinkNotFoundError("No valid element named " + name)

file_path: mechanicalsoup/form.py
snippet:     def new_control(self, type, name, value, **kwargs):
        """Add a new input element to the form.

        The arguments set the attributes of the new element.
        """
        # Remove existing input-like elements with the same name
        for tag in ('input', 'textarea', 'select'):
            for old in self.form.find_all(tag, {'name': name}):
                old.decompose()
        # We don't have access to the original soup object (just the
        # Tag), so we instantiate a new BeautifulSoup() to call
        # new_tag(). We're only building the soup object, not parsing
        # anything, so the parser doesn't matter. Specify the one
        # included in Python to avoid having dependency issue.
        control = BeautifulSoup("", "html.parser").new_tag('input')
        control['type'] = type
        control['name'] = name
        control['value'] = value
        for k, v in kwargs.items():
            control[k] = v
        self._assert_valid_file_upload(control, value)
        self.form.append(control)
        return control

file_path: mechanicalsoup/utils.py
snippet: def is_multipart_file_upload(form, tag):
    return (
        form.get("enctype", "") == "multipart/form-data" and
        tag.get("type", "").lower() == "file"
    )

file_path: mechanicalsoup/form.py
snippet:     def _assert_valid_file_upload(self, tag, value):
        """Raise an exception if a multipart file input is not an open file."""
        if (
            is_multipart_file_upload(self.form, tag) and
            not isinstance(value, io.IOBase)
        ):
            raise ValueError(
                "From v1.3.0 onwards, you must pass an open file object "
                'directly, e.g. `form["name"] = open("/path/to/file", "rb")`. '
                "This change is to remediate a security vulnerability where "
                "a malicious web server could read arbitrary files from the "
                "client (CVE-2023-34457)."
            )