Repository: pallets/click
Pull request #2592 (2023-08-20 16:15:23)

deprecate `OptionParser`
The first step toward rewriting the parser #2205. The `parser` module, `OptionParser`, `Parameter.add_to_parser`, and `Context.protected_args` are all deprecated. Imports will show deprecation warnings.


Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 076dc2b..3c59d86 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -13,6 +13,16 @@ Unreleased
     commands. :issue:`2589`
 -   ``MultiCommand`` is deprecated. ``Group`` is the base class for all group
     commands. :issue:`2590`
+-   The current parser and related classes and methods, are deprecated.
+    :issue:`2205`
+
+    -   ``OptionParser`` and the ``parser`` module, which is a modified copy of
+        ``optparse`` in the standard library.
+    -   ``Context.protected_args`` is unneeded. ``Context.args`` contains any
+        remaining arguments while parsing.
+    -   ``Parameter.add_to_parser`` (on both ``Argument`` and ``Option``) is
+        unneeded. Parsing works directly without building a separate parser.
+    -   ``split_arg_string`` is moved from ``parser`` to ``shell_completion``.
 
 
 Version 8.1.7
diff --git a/src/click/__init__.py b/src/click/__init__.py
index d611adc..c714f55 100644
--- a/src/click/__init__.py
+++ b/src/click/__init__.py
@@ -34,7 +34,6 @@ from .exceptions import UsageError as UsageError
 from .formatting import HelpFormatter as HelpFormatter
 from .formatting import wrap_text as wrap_text
 from .globals import get_current_context as get_current_context
-from .parser import OptionParser as OptionParser
 from .termui import clear as clear
 from .termui import confirm as confirm
 from .termui import echo_via_pager as echo_via_pager
@@ -96,4 +95,15 @@ def __getattr__(name: str) -> object:
         )
         return _MultiCommand
 
+    if name == "OptionParser":
+        from .parser import _OptionParser
+
+        warnings.warn(
+            "'OptionParser' is deprecated and will be removed in Click 9.0. The"
+            " old parser is available in 'optparse'.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return _OptionParser
+
     raise AttributeError(name)
diff --git a/src/click/core.py b/src/click/core.py
index 827e99b..c5900df 100644
--- a/src/click/core.py
+++ b/src/click/core.py
@@ -25,8 +25,8 @@ from .formatting import join_options
 from .globals import pop_context
 from .globals import push_context
 from .parser import _flag_needs_value
-from .parser import OptionParser
-from .parser import split_opt
+from .parser import _OptionParser
+from .parser import _split_opt
 from .termui import confirm
 from .termui import prompt
 from .termui import style
@@ -224,6 +224,10 @@ class Context:
         context. ``Command.show_default`` overrides this default for the
         specific command.
 
+    .. versionchanged:: 8.2
+        The ``protected_args`` attribute is deprecated and will be removed in
+        Click 9.0. ``args`` will contain remaining unparsed tokens.
+
     .. versionchanged:: 8.1
         The ``show_default`` parameter is overridden by
         ``Command.show_default``, instead of the other way around.
@@ -287,7 +291,7 @@ class Context:
         #: to `args` when certain parsing scenarios are encountered but
         #: must be never propagated to another arguments.  This is used
         #: to implement nested parsing.
-        self.protected_args: t.List[str] = []
+        self._protected_args: t.List[str] = []
         #: the collected prefixes of the command's options.
         self._opt_prefixes: t.Set[str] = set(parent._opt_prefixes) if parent else set()
 
@@ -425,6 +429,18 @@ class Context:
         self._parameter_source: t.Dict[str, ParameterSource] = {}
         self._exit_stack = ExitStack()
 
+    @property
+    def protected_args(self) -> t.List[str]:
+        import warnings
+
+        warnings.warn(
+            "'protected_args' is deprecated and will be removed in Click 9.0."
+            " 'args' will contain remaining unparsed tokens.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self._protected_args
+
     def to_info_dict(self) -> t.Dict[str, t.Any]:
         """Gather information that could be useful for a tool generating
         user-facing documentation. This traverses the entire CLI
@@ -1009,9 +1025,9 @@ class Command:
             help=_("Show this message and exit."),
         )
 
-    def make_parser(self, ctx: Context) -> OptionParser:
+    def make_parser(self, ctx: Context) -> _OptionParser:
         """Creates the underlying option parser for this command."""
-        parser = OptionParser(ctx)
+        parser = _OptionParser(ctx)
         for param in self.get_params(ctx):
             param.add_to_parser(parser, ctx)
         return parser
@@ -1212,7 +1228,7 @@ class Command:
                 results.extend(
                     CompletionItem(name, help=command.get_short_help_str())
                     for name, command in _complete_visible_commands(ctx, incomplete)
-                    if name not in ctx.protected_args
+                    if name not in ctx._protected_args
                 )
 
         return results
@@ -1738,10 +1754,10 @@ class Group(Command):
         rest = super().parse_args(ctx, args)
 
         if self.chain:
-            ctx.protected_args = rest
+            ctx._protected_args = rest
             ctx.args = []
         elif rest:
-            ctx.protected_args, ctx.args = rest[:1], rest[1:]
+            ctx._protected_args, ctx.args = rest[:1], rest[1:]
 
         return ctx.args
 
@@ -1751,7 +1767,7 @@ class Group(Command):
                 value = ctx.invoke(self._result_callback, value, **ctx.params)
             return value
 
-        if not ctx.protected_args:
+        if not ctx._protected_args:
             if self.invoke_without_command:
                 # No subcommand was invoked, so the result callback is
                 # invoked with the group return value for regular
@@ -1762,9 +1778,9 @@ class Group(Command):
             ctx.fail(_("Missing command."))
 
         # Fetch args back out
-        args = [*ctx.protected_args, *ctx.args]
+        args = [*ctx._protected_args, *ctx.args]
         ctx.args = []
-        ctx.protected_args = []
+        ctx._protected_args = []
 
         # If we're not in chain mode, we only allow the invocation of a
         # single command but we also inform the current context about the
@@ -1835,7 +1851,7 @@ class Group(Command):
         # resolve things like --help which now should go to the main
         # place.
         if cmd is None and not ctx.resilient_parsing:
-            if split_opt(cmd_name)[0]:
+            if _split_opt(cmd_name)[0]:
                 self.parse_args(ctx, ctx.args)
             ctx.fail(_("No such command {name!r}.").format(name=original_cmd_name))
         return cmd_name if cmd else None, cmd, args[1:]
@@ -2193,7 +2209,7 @@ class Parameter:
 
         return value
 
-    def add_to_parser(self, parser: OptionParser, ctx: Context) -> None:
+    def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None:
         raise NotImplementedError()
 
     def consume_value(
@@ -2582,7 +2598,7 @@ class Option(Parameter):
                     first, second = decl.split(split_char, 1)
                     first = first.rstrip()
                     if first:
-                        possible_names.append(split_opt(first))
+                        possible_names.append(_split_opt(first))
                         opts.append(first)
                     second = second.lstrip()
                     if second:
@@ -2593,7 +2609,7 @@ class Option(Parameter):
                             " same flag for true/false."
                         )
                 else:
-                    possible_names.append(split_opt(decl))
+                    possible_names.append(_split_opt(decl))
                     opts.append(decl)
 
         if name is None and possible_names:
@@ -2616,7 +2632,7 @@ class Option(Parameter):
 
         return name, opts, secondary_opts
 
-    def add_to_parser(self, parser: OptionParser, ctx: Context) -> None:
+    def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None:
         if self.multiple:
             action = "append"
         elif self.count:
@@ -2733,7 +2749,7 @@ class Option(Parameter):
             elif self.is_bool_flag and self.secondary_opts:
                 # For boolean flags that have distinct True/False opts,
                 # use the opt without prefix instead of the value.
-                default_string = split_opt(
+                default_string = _split_opt(
                     (self.opts if self.default else self.secondary_opts)[0]
                 )[1]
             elif self.is_bool_flag and not self.secondary_opts and not default_value:
@@ -2962,7 +2978,7 @@ class Argument(Parameter):
     def get_error_hint(self, ctx: Context) -> str:
         return f"'{self.make_metavar()}'"
 
-    def add_to_parser(self, parser: OptionParser, ctx: Context) -> None:
+    def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None:
         parser.add_argument(dest=self.name, nargs=self.nargs, obj=self)
 
 
diff --git a/src/click/formatting.py b/src/click/formatting.py
index ddd2a2f..7c316b6 100644
--- a/src/click/formatting.py
+++ b/src/click/formatting.py
@@ -3,7 +3,7 @@ from contextlib import contextmanager
 from gettext import gettext as _
 
 from ._compat import term_len
-from .parser import split_opt
+from .parser import _split_opt
 
 # Can force a width.  This is used by the test system
 FORCED_WIDTH: t.Optional[int] = None
@@ -290,7 +290,7 @@ def join_options(options: t.Sequence[str]) -> t.Tuple[str, bool]:
     any_prefix_is_slash = False
 
     for opt in options:
-        prefix = split_opt(opt)[0]
+        prefix = _split_opt(opt)[0]
 
         if prefix == "/":
             any_prefix_is_slash = True
diff --git a/src/click/parser.py b/src/click/parser.py
index 5fa7adf..643ad7e 100644
--- a/src/click/parser.py
+++ b/src/click/parser.py
@@ -106,7 +106,7 @@ def _unpack_args(
     return tuple(rv), list(args)
 
 
-def split_opt(opt: str) -> t.Tuple[str, str]:
+def _split_opt(opt: str) -> t.Tuple[str, str]:
     first = opt[:1]
     if first.isalnum():
         return "", opt
@@ -115,48 +115,14 @@ def split_opt(opt: str) -> t.Tuple[str, str]:
     return first, opt[1:]
 
 
-def normalize_opt(opt: str, ctx: t.Optional["Context"]) -> str:
+def _normalize_opt(opt: str, ctx: t.Optional["Context"]) -> str:
     if ctx is None or ctx.token_normalize_func is None:
         return opt
-    prefix, opt = split_opt(opt)
+    prefix, opt = _split_opt(opt)
     return f"{prefix}{ctx.token_normalize_func(opt)}"
 
 
-def split_arg_string(string: str) -> t.List[str]:
-    """Split an argument string as with :func:`shlex.split`, but don't
-    fail if the string is incomplete. Ignores a missing closing quote or
-    incomplete escape sequence and uses the partial token as-is.
-
-    .. code-block:: python
-
-        split_arg_string("example 'my file")
-        ["example", "my file"]
-
-        split_arg_string("example my\\")
-        ["example", "my"]
-
-    :param string: String to split.
-    """
-    import shlex
-
-    lex = shlex.shlex(string, posix=True)
-    lex.whitespace_split = True
-    lex.commenters = ""
-    out = []
-
-    try:
-        for token in lex:
-            out.append(token)
-    except ValueError:
-        # Raised when end-of-string is reached in an invalid state. Use
-        # the partial token as-is. The quote or escape character is in
-        # lex.state, not lex.token.
-        out.append(lex.token)
-
-    return out
-
-
-class Option:
+class _Option:
     def __init__(
         self,
         obj: "CoreOption",
@@ -171,7 +137,7 @@ class Option:
         self.prefixes: t.Set[str] = set()
 
         for opt in opts:
-            prefix, value = split_opt(opt)
+            prefix, value = _split_opt(opt)
             if not prefix:
                 raise ValueError(f"Invalid start character for option ({opt})")
             self.prefixes.add(prefix[0])
@@ -194,7 +160,7 @@ class Option:
     def takes_value(self) -> bool:
         return self.action in ("store", "append")
 
-    def process(self, value: t.Any, state: "ParsingState") -> None:
+    def process(self, value: t.Any, state: "_ParsingState") -> None:
         if self.action == "store":
             state.opts[self.dest] = value  # type: ignore
         elif self.action == "store_const":
@@ -210,7 +176,7 @@ class Option:
         state.order.append(self.obj)
 
 
-class Argument:
+class _Argument:
     def __init__(self, obj: "CoreArgument", dest: t.Optional[str], nargs: int = 1):
         self.dest = dest
         self.nargs = nargs
@@ -219,7 +185,7 @@ class Argument:
     def process(
         self,
         value: t.Union[t.Optional[str], t.Sequence[t.Optional[str]]],
-        state: "ParsingState",
+        state: "_ParsingState",
     ) -> None:
         if self.nargs > 1:
             assert value is not None
@@ -242,7 +208,7 @@ class Argument:
         state.order.append(self.obj)
 
 
-class ParsingState:
+class _ParsingState:
     def __init__(self, rargs: t.List[str]) -> None:
         self.opts: t.Dict[str, t.Any] = {}
         self.largs: t.List[str] = []
@@ -250,7 +216,7 @@ class ParsingState:
         self.order: t.List["CoreParameter"] = []
 
 
-class OptionParser:
+class _OptionParser:
     """The option parser is an internal class that is ultimately used to
     parse options and arguments.  It's modelled after optparse and brings
     a similar but vastly simplified API.  It should generally not be used
@@ -262,6 +228,9 @@ class OptionParser:
 
     :param ctx: optionally the :class:`~click.Context` where this parser
                 should go with.
+
+    .. deprecated:: 8.2
+        Will be removed in Click 9.0.
     """
 
     def __init__(self, ctx: t.Optional["Context"] = None) -> None:
@@ -283,10 +252,10 @@ class OptionParser:
             self.allow_interspersed_args = ctx.allow_interspersed_args
             self.ignore_unknown_options = ctx.ignore_unknown_options
 
-        self._short_opt: t.Dict[str, Option] = {}
-        self._long_opt: t.Dict[str, Option] = {}
+        self._short_opt: t.Dict[str, _Option] = {}
+        self._long_opt: t.Dict[str, _Option] = {}
         self._opt_prefixes = {"-", "--"}
-        self._args: t.List[Argument] = []
+        self._args: t.List[_Argument] = []
 
     def add_option(
         self,
@@ -305,8 +274,8 @@ class OptionParser:
         The `obj` can be used to identify the option in the order list
         that is returned from the parser.
         """
-        opts = [normalize_opt(opt, self.ctx) for opt in opts]
-        option = Option(obj, opts, dest, action=action, nargs=nargs, const=const)
+        opts = [_normalize_opt(opt, self.ctx) for opt in opts]
+        option = _Option(obj, opts, dest, action=action, nargs=nargs, const=const)
         self._opt_prefixes.update(option.prefixes)
         for opt in option._short_opts:
             self._short_opt[opt] = option
@@ -321,7 +290,7 @@ class OptionParser:
         The `obj` can be used to identify the option in the order list
         that is returned from the parser.
         """
-        self._args.append(Argument(obj, dest=dest, nargs=nargs))
+        self._args.append(_Argument(obj, dest=dest, nargs=nargs))
 
     def parse_args(
         self, args: t.List[str]
@@ -332,7 +301,7 @@ class OptionParser:
         appear on the command line.  If arguments appear multiple times they
         will be memorized multiple times as well.
         """
-        state = ParsingState(args)
+        state = _ParsingState(args)
         try:
             self._process_args_for_options(state)
             self._process_args_for_args(state)
@@ -341,7 +310,7 @@ class OptionParser:
                 raise
         return state.opts, state.largs, state.order
 
-    def _process_args_for_args(self, state: ParsingState) -> None:
+    def _process_args_for_args(self, state: _ParsingState) -> None:
         pargs, args = _unpack_args(
             state.largs + state.rargs, [x.nargs for x in self._args]
         )
@@ -352,7 +321,7 @@ class OptionParser:
         state.largs = args
         state.rargs = []
 
-    def _process_args_for_options(self, state: ParsingState) -> None:
+    def _process_args_for_options(self, state: _ParsingState) -> None:
         while state.rargs:
             arg = state.rargs.pop(0)
             arglen = len(arg)
@@ -389,7 +358,7 @@ class OptionParser:
         # not a very interesting subset!
 
     def _match_long_opt(
-        self, opt: str, explicit_value: t.Optional[str], state: ParsingState
+        self, opt: str, explicit_value: t.Optional[str], state: _ParsingState
     ) -> None:
         if opt not in self._long_opt:
             from difflib import get_close_matches
@@ -418,14 +387,14 @@ class OptionParser:
 
         option.process(value, state)
 
-    def _match_short_opt(self, arg: str, state: ParsingState) -> None:
+    def _match_short_opt(self, arg: str, state: _ParsingState) -> None:
         stop = False
         i = 1
         prefix = arg[0]
         unknown_options = []
 
         for ch in arg[1:]:
-            opt = normalize_opt(f"{prefix}{ch}", self.ctx)
+            opt = _normalize_opt(f"{prefix}{ch}", self.ctx)
             option = self._short_opt.get(opt)
             i += 1
 
@@ -459,7 +428,7 @@ class OptionParser:
             state.largs.append(f"{prefix}{''.join(unknown_options)}")
 
     def _get_value_from_state(
-        self, option_name: str, option: Option, state: ParsingState
+        self, option_name: str, option: _Option, state: _ParsingState
     ) -> t.Any:
         nargs = option.nargs
 
@@ -496,7 +465,7 @@ class OptionParser:
 
         return value
 
-    def _process_opts(self, arg: str, state: ParsingState) -> None:
+    def _process_opts(self, arg: str, state: _ParsingState) -> None:
         explicit_value = None
         # Long option handling happens in two parts.  The first part is
         # supporting explicitly attached values.  In any case, we will try
@@ -505,7 +474,7 @@ class OptionParser:
             long_opt, explicit_value = arg.split("=", 1)
         else:
             long_opt = arg
-        norm_long_opt = normalize_opt(long_opt, self.ctx)
+        norm_long_opt = _normalize_opt(long_opt, self.ctx)
 
         # At this point we will match the (assumed) long option through
         # the long option matching code.  Note that this allows options
@@ -527,3 +496,34 @@ class OptionParser:
                 raise
 
             state.largs.append(arg)
+
+
+def __getattr__(name: str) -> object:
+    import warnings
+
+    if name in {
+        "OptionParser",
+        "Argument",
+        "Option",
+        "split_opt",
+        "normalize_opt",
+        "ParsingState",
+    }:
+        warnings.warn(
+            f"'parser.{name}' is deprecated and will be removed in Click 9.0."
+            " The old parser is available in 'optparse'."
+        )
+        return globals()[f"_{name}"]
+
+    if name == "split_arg_string":
+        from .shell_completion import split_arg_string
+
+        warnings.warn(
+            "Importing 'parser.split_arg_string' is deprecated, it will only be"
+            " available in 'shell_completion' in Click 9.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return split_arg_string
+
+    raise AttributeError(name)
diff --git a/src/click/shell_completion.py b/src/click/shell_completion.py
index b2d653e..2edafda 100644
--- a/src/click/shell_completion.py
+++ b/src/click/shell_completion.py
@@ -10,7 +10,6 @@ from .core import Group
 from .core import Option
 from .core import Parameter
 from .core import ParameterSource
-from .parser import split_arg_string
 from .utils import echo
 
 
@@ -434,6 +433,43 @@ def get_completion_class(shell: str) -> t.Optional[t.Type[ShellComplete]]:
     return _available_shells.get(shell)
 
 
+def split_arg_string(string: str) -> t.List[str]:
+    """Split an argument string as with :func:`shlex.split`, but don't
+    fail if the string is incomplete. Ignores a missing closing quote or
+    incomplete escape sequence and uses the partial token as-is.
+
+    .. code-block:: python
+
+        split_arg_string("example 'my file")
+        ["example", "my file"]
+
+        split_arg_string("example my\\")
+        ["example", "my"]
+
+    :param string: String to split.
+
+    .. versionchanged:: 8.2
+        Moved to ``shell_completion`` from ``parser``.
+    """
+    import shlex
+
+    lex = shlex.shlex(string, posix=True)
+    lex.whitespace_split = True
+    lex.commenters = ""
+    out = []
+
+    try:
+        for token in lex:
+            out.append(token)
+    except ValueError:
+        # Raised when end-of-string is reached in an invalid state. Use
+        # the partial token as-is. The quote or escape character is in
+        # lex.state, not lex.token.
+        out.append(lex.token)
+
+    return out
+
+
 def _is_incomplete_argument(ctx: Context, param: Parameter) -> bool:
     """Determine if the given parameter is an argument that can still
     accept values.
@@ -508,7 +544,7 @@ def _resolve_context(
     """
     ctx_args["resilient_parsing"] = True
     ctx = cli.make_context(prog_name, args.copy(), **ctx_args)
-    args = ctx.protected_args + ctx.args
+    args = ctx._protected_args + ctx.args
 
     while args:
         command = ctx.command
@@ -521,7 +557,7 @@ def _resolve_context(
                     return ctx
 
                 ctx = cmd.make_context(name, args, parent=ctx, resilient_parsing=True)
-                args = ctx.protected_args + ctx.args
+                args = ctx._protected_args + ctx.args
             else:
                 sub_ctx = ctx
 
@@ -542,7 +578,7 @@ def _resolve_context(
                     args = sub_ctx.args
 
                 ctx = sub_ctx
-                args = [*sub_ctx.protected_args, *sub_ctx.args]
+                args = [*sub_ctx._protected_args, *sub_ctx.args]
         else:
             break
 


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
index f694916..f2a3ad5 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1,8 +1,8 @@
 import pytest
 
 import click
-from click.parser import OptionParser
-from click.parser import split_arg_string
+from click.parser import _OptionParser
+from click.shell_completion import split_arg_string
 
 
 @pytest.mark.parametrize(
@@ -20,13 +20,13 @@ def test_split_arg_string(value, expect):
 
 
 def test_parser_default_prefixes():
-    parser = OptionParser()
+    parser = _OptionParser()
     assert parser._opt_prefixes == {"-", "--"}
 
 
 def test_parser_collects_prefixes():
     ctx = click.Context(click.Command("test"))
-    parser = OptionParser(ctx)
+    parser = _OptionParser(ctx)
     click.Option("+p", is_flag=True).add_to_parser(parser, ctx)
     click.Option("!e", is_flag=True).add_to_parser(parser, ctx)
     assert parser._opt_prefixes == {"-", "--", "+", "!"}