Repository: scrapy/scrapy
Pull request #5902 (2023-04-19 05:08:20)

fix: docs/intro/tutorial.rst checks
Fix docs/intro/tutorial.rst checks:

command:
```
pytest docs
```

output:
```
FAILED docs/intro/tutorial.rst::line:254,column:1
FAILED docs/intro/tutorial.rst::line:350,column:1
FAILED docs/intro/tutorial.rst::line:412,column:1
```


Patch:
diff --git a/docs/faq.rst b/docs/faq.rst
index 031f4b942..20dd814df 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -231,8 +231,8 @@ Can I return (Twisted) deferreds from signal handlers?
 Some signals support returning deferreds from their handlers, others don't. See
 the :ref:`topics-signals-ref` to know which ones.
 
-What does the response status code 999 means?
----------------------------------------------
+What does the response status code 999 mean?
+--------------------------------------------
 
 999 is a custom response status code used by Yahoo sites to throttle requests.
 Try slowing down the crawling speed by using a download delay of ``2`` (or
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 04d73d95a..19a76fc16 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -252,7 +252,7 @@ object:
 .. code-block:: pycon
 
     >>> response.css("title")
-    [<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
+    [<Selector query='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
 
 The result of running ``response.css('title')`` is a list-like object called
 :class:`~scrapy.selector.SelectorList`, which represents a list of
@@ -348,7 +348,7 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:
 .. code-block:: pycon
 
     >>> response.xpath("//title")
-    [<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
+    [<Selector query='//title' data='<title>Quotes to Scrape</title>'>]
     >>> response.xpath("//title/text()").get()
     'Quotes to Scrape'
 
@@ -410,8 +410,8 @@ We get a list of selectors for the quote HTML elements with:
 .. code-block:: pycon
 
     >>> response.css("div.quote")
-    [<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
-    <Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
+    [<Selector query="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
+    <Selector query="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
     ...]
 
 Each of the selectors returned by the query above allows us to run further
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index a2243ae2e..f5dbbd64e 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -7,6 +7,14 @@ from urllib.parse import urlparse
 from w3lib.http import basic_auth_header
 
 
+class DataAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        value = str(values)
+        if value.startswith("$"):
+            value = value[1:]
+        setattr(namespace, self.dest, value)
+
+
 class CurlParser(argparse.ArgumentParser):
     def error(self, message):
         error_msg = f"There was an error parsing the curl command: {message}"
@@ -17,7 +25,7 @@ curl_parser = CurlParser()
 curl_parser.add_argument("url")
 curl_parser.add_argument("-H", "--header", dest="headers", action="append")
 curl_parser.add_argument("-X", "--request", dest="method")
-curl_parser.add_argument("-d", "--data", "--data-raw", dest="data")
+curl_parser.add_argument("-d", "--data", "--data-raw", dest="data", action=DataAction)
 curl_parser.add_argument("-u", "--user", dest="auth")
 
 
diff --git a/tox.ini b/tox.ini
index 06b52f3dc..9d81ec3e7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -30,7 +30,7 @@ passenv =
 #allow tox virtualenv to upgrade pip/wheel/setuptools
 download = true
 commands =
-    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}
+    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules
 install_command =
     python -I -m pip install -ctests/upper-constraints.txt {opts} {packages}
 
@@ -99,6 +99,8 @@ setenv =
     _SCRAPY_PINNED=true
 install_command =
     python -I -m pip install {opts} {packages}
+commands =
+    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests}
 
 [testenv:pinned]
 basepython = python3.7
@@ -108,6 +110,7 @@ deps =
 install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}
+commands = {[pinned]commands}
 
 [testenv:windows-pinned]
 basepython = python3
@@ -117,6 +120,7 @@ deps =
 install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}
+commands = {[pinned]commands}
 
 [testenv:extra-deps]
 basepython = python3
@@ -137,7 +141,7 @@ commands =
 
 [testenv:asyncio-pinned]
 deps = {[testenv:pinned]deps}
-commands = {[testenv:asyncio]commands}
+commands = {[pinned]commands} --reactor=asyncio
 install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}
@@ -152,7 +156,8 @@ basepython = {[testenv:pypy3]basepython}
 deps =
     {[pinned]deps}
     PyPyDispatcher==2.1.0
-commands = {[testenv:pypy3]commands}
+commands =
+    pytest --durations=10 scrapy tests
 install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}


Test patch:
diff --git a/pytest.ini b/pytest.ini
index f5fbf2529..866f0c950 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -5,7 +5,6 @@ python_files=test_*.py __init__.py
 python_classes=
 addopts =
     --assert=plain
-    --doctest-modules
     --ignore=docs/_ext
     --ignore=docs/conf.py
     --ignore=docs/news.rst
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 3124d9d67..83de0e77e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -2542,7 +2542,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
     def test_batch_path_differ(self):
         """
         Test that the name of all batch files differ from each other.
-        So %(batch_time)s replaced with the current date.
+        So %(batch_id)d replaced with the current id.
         """
         items = [
             self.MyItem({"foo": "bar1", "egg": "spam1"}),
@@ -2552,7 +2552,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
         settings = {
             "FEEDS": {
                 self._random_temp_filename()
-                / "%(batch_time)s": {
+                / "%(batch_id)d": {
                     "format": "json",
                 },
             },
@@ -2615,7 +2615,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 return super().open(*args, **kwargs)
 
         key = "export.csv"
-        uri = f"s3://{bucket}/{key}/%(batch_time)s.json"
+        uri = f"s3://{bucket}/{key}/%(batch_id)d.json"
         batch_item_count = 1
         settings = {
             "AWS_ACCESS_KEY_ID": "access_key",
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index fd4612eba..1816db29b 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -154,6 +154,15 @@ class CurlToRequestKwargsTest(unittest.TestCase):
         }
         self._test_command(curl_command, expected_result)
 
+    def test_post_data_raw_with_string_prefix(self):
+        curl_command = "curl 'https://www.example.org/' --data-raw $'{\"$filters\":\"Filter\u0021\"}'"
+        expected_result = {
+            "method": "POST",
+            "url": "https://www.example.org/",
+            "body": '{"$filters":"Filter!"}',
+        }
+        self._test_command(curl_command, expected_result)
+
     def test_explicit_get_with_data(self):
         curl_command = "curl httpbin.org/anything -X GET --data asdf"
         expected_result = {