Repository: psf/black

Issue:
Add ESP to --preview for stable
A preview flag is added in #2751, which could also contain experimental string processing. So shall we convert ESP there? Or are we still thinking about making it stable very soon?


Discussion:
I don't think there's enough time to turn it on by default in this month's release. We should include it in the `--preview` feature set though.
Do we want to keep the existing ESP flag simultaneously with its `--preview` addition?

Personally, I'd want to remove it and a first stable release seems like a good excuse, but Jelle raised a good point about keeping compatibility on Discord. This could also affect the feature flag machinery implementation in #2752.
I would be fine with removing the ESP flag as we are doing a major stable release, so "breaking" a bit of compatibility is fine.

If we decide to keep it for compatibility reasons, we could deprecate the flag and add some sort of "warning" log when someone uses an ESP flag. The deprecation could either still preserve the flag's functionality or direct the user to the new `--preview` flag (without actually doing anything, just outputting a simple message).
Hmm, redirecting users sounds even better! I'm thinking: catch the use of ESP and throw before doing anything with a message telling people to use `--preview`.

But a softer deprecation works as well!
I disagree with the idea that "we're doing a stable release, so we can break things". We've spent too many years in "beta" to keep that distinction meaningful.

My preference is to keep the behavior of the existing flag the same, but deprecate it and fold it into `--preview` in a few releases.
> catch the use of ESP and throw before doing anything with a message telling people to use `--preview`.

Yeah, that's what I meant 👍🏻 

> My preference is to keep the behavior of the existing flag the same, but deprecate it and fold it into `--preview` in a few releases.

By few releases, do we mean major releases or `22.X`?
I imagine yes, in a few minor releases. "In a few years" sounds too much 😄 So we'll do the soft deprecation! I'll work on it later today.
There is no such thing as a major or minor release on this project. We use calendar versioning.
Yeah, I meant the second number place, I don't really know what they are called in calver. Anyways, as you said, "in a couple of releases".

Resolving patch:
diff --git a/src/black/__init__.py b/src/black/__init__.py
--- a/src/black/__init__.py
+++ b/src/black/__init__.py
@@ -241,10 +241,7 @@ def validate_regex(
     "--experimental-string-processing",
     is_flag=True,
     hidden=True,
-    help=(
-        "Experimental option that performs more normalization on string literals."
-        " Currently disabled because it leads to some crashes."
-    ),
+    help="(DEPRECATED and now included in --preview) Normalize string literals.",
 )
 @click.option(
     "--preview",
diff --git a/src/black/linegen.py b/src/black/linegen.py
--- a/src/black/linegen.py
+++ b/src/black/linegen.py
@@ -23,8 +23,7 @@
 from black.strings import normalize_string_prefix, normalize_string_quotes
 from black.trans import Transformer, CannotTransform, StringMerger
 from black.trans import StringSplitter, StringParenWrapper, StringParenStripper
-from black.mode import Mode
-from black.mode import Feature
+from black.mode import Mode, Feature, Preview
 
 from blib2to3.pytree import Node, Leaf
 from blib2to3.pgen2 import token
@@ -338,7 +337,7 @@ def transform_line(
         and not (line.inside_brackets and line.contains_standalone_comments())
     ):
         # Only apply basic string preprocessing, since lines shouldn't be split here.
-        if mode.experimental_string_processing:
+        if Preview.string_processing in mode:
             transformers = [string_merge, string_paren_strip]
         else:
             transformers = []
@@ -381,7 +380,7 @@ def _rhs(
         # via type ... https://github.com/mypyc/mypyc/issues/884
         rhs = type("rhs", (), {"__call__": _rhs})()
 
-        if mode.experimental_string_processing:
+        if Preview.string_processing in mode:
             if line.inside_brackets:
                 transformers = [
                     string_merge,
diff --git a/src/black/mode.py b/src/black/mode.py
--- a/src/black/mode.py
+++ b/src/black/mode.py
@@ -7,9 +7,10 @@
 import sys
 
 from dataclasses import dataclass, field
-from enum import Enum
+from enum import Enum, auto
 from operator import attrgetter
 from typing import Dict, Set
+from warnings import warn
 
 if sys.version_info < (3, 8):
     from typing_extensions import Final
@@ -124,6 +125,13 @@ def supports_feature(target_versions: Set[TargetVersion], feature: Feature) -> b
 class Preview(Enum):
     """Individual preview style features."""
 
+    string_processing = auto()
+    hug_simple_powers = auto()
+
+
+class Deprecated(UserWarning):
+    """Visible deprecation warning."""
+
 
 @dataclass
 class Mode:
@@ -136,6 +144,14 @@ class Mode:
     experimental_string_processing: bool = False
     preview: bool = False
 
+    def __post_init__(self) -> None:
+        if self.experimental_string_processing:
+            warn(
+                "`experimental string processing` has been included in `preview`"
+                " and deprecated. Use `preview` instead.",
+                Deprecated,
+            )
+
     def __contains__(self, feature: Preview) -> bool:
         """
         Provide `Preview.FEATURE in Mode` syntax that mirrors the ``preview`` flag.
@@ -143,6 +159,8 @@ def __contains__(self, feature: Preview) -> bool:
         The argument is not checked and features are not differentiated.
         They only exist to make development easier by clarifying intent.
         """
+        if feature is Preview.string_processing:
+            return self.preview or self.experimental_string_processing
         return self.preview
 
     def get_cache_key(self) -> str:


Test patch:
diff --git a/tests/test_black.py b/tests/test_black.py
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -150,6 +150,11 @@ def test_empty_ff(self) -> None:
             os.unlink(tmp_file)
         self.assertFormatEqual(expected, actual)
 
+    def test_experimental_string_processing_warns(self) -> None:
+        self.assertWarns(
+            black.mode.Deprecated, black.Mode, experimental_string_processing=True
+        )
+
     def test_piping(self) -> None:
         source, expected = read_data("src/black/__init__", data=False)
         result = BlackRunner().invoke(
@@ -342,7 +347,7 @@ def test_detect_pos_only_arguments(self) -> None:
     @patch("black.dump_to_file", dump_to_stderr)
     def test_string_quotes(self) -> None:
         source, expected = read_data("string_quotes")
-        mode = black.Mode(experimental_string_processing=True)
+        mode = black.Mode(preview=True)
         assert_format(source, expected, mode)
         mode = replace(mode, string_normalization=False)
         not_normalized = fs(source, mode=mode)
diff --git a/tests/test_format.py b/tests/test_format.py
--- a/tests/test_format.py
+++ b/tests/test_format.py
@@ -55,15 +55,6 @@
     "tupleassign",
 ]
 
-EXPERIMENTAL_STRING_PROCESSING_CASES: List[str] = [
-    "cantfit",
-    "comments7",
-    "long_strings",
-    "long_strings__edge_case",
-    "long_strings__regression",
-    "percent_precedence",
-]
-
 PY310_CASES: List[str] = [
     "pattern_matching_simple",
     "pattern_matching_complex",
@@ -73,7 +64,15 @@
     "parenthesized_context_managers",
 ]
 
-PREVIEW_CASES: List[str] = []
+PREVIEW_CASES: List[str] = [
+    # string processing
+    "cantfit",
+    "comments7",
+    "long_strings",
+    "long_strings__edge_case",
+    "long_strings__regression",
+    "percent_precedence",
+]
 
 SOURCES: List[str] = [
     "src/black/__init__.py",
@@ -136,11 +135,6 @@ def test_simple_format(filename: str) -> None:
     check_file(filename, DEFAULT_MODE)
 
 
-@pytest.mark.parametrize("filename", EXPERIMENTAL_STRING_PROCESSING_CASES)
-def test_experimental_format(filename: str) -> None:
-    check_file(filename, black.Mode(experimental_string_processing=True))
-
-
 @pytest.mark.parametrize("filename", PREVIEW_CASES)
 def test_preview_format(filename: str) -> None:
     check_file(filename, black.Mode(preview=True))