nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: scrapy/scrapy
Pull request #1185 (2015-04-21 19:30:27)

[MRG +1]fixed backwards compatibility for SPIDER_MANAGER_CLASS option
Fixed an issue with https://github.com/scrapy/scrapy/pull/1166 which @dangra spotted.


Patch:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index dab1043ec..b4706919a 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -192,8 +192,8 @@ def _get_spider_loader(settings):
             'Please use SPIDER_LOADER_CLASS.',
             category=ScrapyDeprecationWarning, stacklevel=2
         )
-    cls_path = settings.get('SPIDER_LOADER_CLASS',
-                            settings.get('SPIDER_MANAGER_CLASS'))
+    cls_path = settings.get('SPIDER_MANAGER_CLASS',
+                            settings.get('SPIDER_LOADER_CLASS'))
     loader_cls = load_object(cls_path)
     verifyClass(ISpiderLoader, loader_cls)
     return loader_cls.from_settings(settings.frozencopy())


Test patch:
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 50da01e73..6a282ed84 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -5,6 +5,7 @@ from zope.interface.verify import DoesNotImplement
 
 from scrapy.crawler import Crawler, CrawlerRunner
 from scrapy.settings import Settings, default_settings
+from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
 
@@ -60,6 +61,10 @@ class SpiderLoaderWithWrongInterface(object):
         pass
 
 
+class CustomSpiderLoader(SpiderLoader):
+    pass
+
+
 class CrawlerRunnerTestCase(unittest.TestCase):
 
     def test_spider_manager_verify_interface(self):
@@ -87,3 +92,13 @@ class CrawlerRunnerTestCase(unittest.TestCase):
             self.assertIn("CrawlerRunner.spider_loader", str(w[0].message))
             sl_cls = load_object(runner.settings['SPIDER_LOADER_CLASS'])
             self.assertIsInstance(spiders, sl_cls)
+
+    def test_spidermanager_deprecation(self):
+        with warnings.catch_warnings(record=True) as w:
+            runner = CrawlerRunner({
+                'SPIDER_MANAGER_CLASS': 'tests.test_crawler.CustomSpiderLoader'
+            })
+            self.assertIsInstance(runner.spider_loader, CustomSpiderLoader)
+            self.assertEqual(len(w), 1)
+            self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message))
+
Repository: scrapy/scrapy
Pull request #1276 (2015-06-02 23:31:45)

Fix Spider.custom_settings
I noticed that a CloseSpider extension doesn't get activated when CLOSESPIDER_PAGECOUNT option is set in spider's `custom_settings`. Other extensions (AutoThrottle, LogStats, etc) are also not activated. 

This happens because extensions check settings in `from_crawler` methods, and `crawler.settings` doesn't contain spider settings yet because `spidercls.update_settings` call was moved to a later stage by #1128.


Patch:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 2cd658276..204395c77 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -29,6 +29,7 @@ class Crawler(object):
 
         self.spidercls = spidercls
         self.settings = settings.copy()
+        self.spidercls.update_settings(self.settings)
 
         self.signals = SignalManager(self)
         self.stats = load_object(self.settings['STATS_CLASS'])(self)
@@ -44,9 +45,7 @@ class Crawler(object):
         self.logformatter = lf_cls.from_crawler(self)
         self.extensions = ExtensionManager.from_crawler(self)
 
-        self.spidercls.update_settings(self.settings)
         self.settings.freeze()
-
         self.crawling = False
         self.spider = None
         self.engine = None


Test patch:
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 53f517c27..1c0804675 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,11 +1,13 @@
 import warnings
 import unittest
 
+import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
+from scrapy.extensions.throttle import AutoThrottle
 
 
 class CrawlerTestCase(unittest.TestCase):
@@ -52,6 +54,18 @@ class CrawlerTestCase(unittest.TestCase):
         self.assertIsInstance(crawler.settings, Settings)
 
 
+class SpiderSettingsTestCase(unittest.TestCase):
+    def test_spider_custom_settings(self):
+        class MySpider(scrapy.Spider):
+            name = 'spider'
+            custom_settings = {
+                'AUTOTHROTTLE_ENABLED': True
+            }
+
+        crawler = Crawler(MySpider, {})
+        enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
+        self.assertIn(AutoThrottle, enabled_exts)
+
 
 class SpiderLoaderWithWrongInterface(object):
Repository: scrapy/scrapy
Pull request #1333 (2015-07-02 18:00:20)

interpreting application/x-json as TextResponse
This content type is found in the ajax pagination on store.apple.com (eg http://store.apple.com/us/search/logitech).

```
ipdb> response.headers['Content-Type']
'application/x-json; encoding=UTF8;charset=UTF-8'
```

Currently gets returned as `Response`.


Patch:
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 460385444..7c017feef 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -24,6 +24,7 @@ class ResponseTypes(object):
         'application/vnd.wap.xhtml+xml': 'scrapy.http.HtmlResponse',
         'application/xml': 'scrapy.http.XmlResponse',
         'application/json': 'scrapy.http.TextResponse',
+        'application/x-json': 'scrapy.http.TextResponse',
         'application/javascript': 'scrapy.http.TextResponse',
         'application/x-javascript': 'scrapy.http.TextResponse',
         'text/xml': 'scrapy.http.XmlResponse',


Test patch:
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 942d22b99..b4083403a 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -35,6 +35,7 @@ class ResponseTypesTest(unittest.TestCase):
             ('application/vnd.wap.xhtml+xml; charset=utf-8', HtmlResponse),
             ('application/xml; charset=UTF-8', XmlResponse),
             ('application/octet-stream', Response),
+            ('application/x-json; encoding=UTF8;charset=UTF-8', TextResponse),
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_type(source)
Repository: scrapy/scrapy
Pull request #1419 (2015-08-10 19:12:15)

Do not fail representing non-http requests


Patch:
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index ac415e508..0487d1e1b 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -79,7 +79,7 @@ def request_httprepr(request):
     parsed = urlparse_cached(request)
     path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
     s = to_bytes(request.method) + b" " + to_bytes(path) + b" HTTP/1.1\r\n"
-    s += b"Host: " + to_bytes(parsed.hostname) + b"\r\n"
+    s += b"Host: " + to_bytes(parsed.hostname or b'') + b"\r\n"
     if request.headers:
         s += request.headers.to_string() + b"\r\n"
     s += b"\r\n"


Test patch:
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index d058bba96..e8a4eb3ea 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -71,5 +71,10 @@ class UtilsRequestTest(unittest.TestCase):
         r1 = Request("http://www.example.com", method='POST', headers={"Content-type": b"text/html"}, body=b"Some body")
         self.assertEqual(request_httprepr(r1), b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body')
 
+    def test_request_httprepr_for_non_http_request(self):
+        # the representation is not important but it must not fail.
+        request_httprepr(Request("file:///tmp/foo.txt"))
+        request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
+
 if __name__ == "__main__":
     unittest.main()
Repository: scrapy/scrapy
Pull request #1447 (2015-08-21 14:32:59)

[MRG +1] Fix RedirectMiddleware not honouring handle_httpstatus meta keys
Fixup for #1364

Makes `RedirectMiddleware` honour the `handle_httpstatus_list` and `handle_httpstatus_all` keys from the `Request.meta` dictionary.


Patch:
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 6d986bbf7..73cc67423 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -724,6 +724,12 @@ responses (and pass them through to your spider) you can do this::
     class MySpider(CrawlSpider):
         handle_httpstatus_list = [301, 302]
 
+The ``handle_httpstatus_list`` key of :attr:`Request.meta
+<scrapy.http.Request.meta>` can also be used to specify which response codes to
+allow on a per-request basis. You can also set the meta key
+``handle_httpstatus_all`` to ``True`` if you want to allow any response code
+for a request.
+
 
 RedirectMiddleware settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 363e56cb8..3cf8d2bee 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -55,7 +55,9 @@ class RedirectMiddleware(BaseRedirectMiddleware):
 
     def process_response(self, request, response, spider):
         if (request.meta.get('dont_redirect', False) or
-               response.status in getattr(spider, 'handle_httpstatus_list', [])):
+               response.status in getattr(spider, 'handle_httpstatus_list', []) or
+               response.status in request.meta.get('handle_httpstatus_list', []) or
+               request.meta.get('handle_httpstatus_all', False)):
             return response
 
         if request.method == 'HEAD':


Test patch:
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 9b00caa51..b3db7c42b 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -139,6 +139,17 @@ class RedirectMiddlewareTest(unittest.TestCase):
         r = self.mw.process_response(req, rsp, smartspider)
         self.assertIs(r, rsp)
 
+    def test_request_meta_handling(self):
+        url = 'http://www.example.com/301'
+        url2 = 'http://www.example.com/redirected'
+        def _test_passthrough(req):
+            rsp = Response(url, headers={'Location': url2}, status=301, request=req)
+            r = self.mw.process_response(req, rsp, self.spider)
+            self.assertIs(r, rsp)
+        _test_passthrough(Request(url, meta={'handle_httpstatus_list':
+                                                           [404, 301, 302]}))
+        _test_passthrough(Request(url, meta={'handle_httpstatus_all': True}))
+
 
 class MetaRefreshMiddlewareTest(unittest.TestCase):
Repository: scrapy/scrapy
Pull request #1503 (2015-09-18 08:19:04)

[MRG+1] interpreting json-amazonui-streaming as TextResponse
Amazon is using this content type to load reviews with ajax:

```
ipdb> response.headers['Content-Type']
'application/json-amazonui-streaming;charset=UTF-8'
```

Don't know if we want to add it since it's Amazon-specific.


Patch:
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 4880cc7b9..319656648 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -25,6 +25,7 @@ class ResponseTypes(object):
         'application/xml': 'scrapy.http.XmlResponse',
         'application/json': 'scrapy.http.TextResponse',
         'application/x-json': 'scrapy.http.TextResponse',
+        'application/json-amazonui-streaming': 'scrapy.http.TextResponse',
         'application/javascript': 'scrapy.http.TextResponse',
         'application/x-javascript': 'scrapy.http.TextResponse',
         'text/xml': 'scrapy.http.XmlResponse',


Test patch:
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 2374d518f..b34147b74 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -36,6 +36,7 @@ class ResponseTypesTest(unittest.TestCase):
             ('application/xml; charset=UTF-8', XmlResponse),
             ('application/octet-stream', Response),
             ('application/x-json; encoding=UTF8;charset=UTF-8', TextResponse),
+            ('application/json-amazonui-streaming;charset=UTF-8', TextResponse),
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_type(source)