nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: scrapy/scrapy
Pull request #3819 (2019-06-07 08:42:56)

[MRG+1] Fix missing values
Issue: https://github.com/scrapy/scrapy/issues/3804


Patch:
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index a7c75a46a..20f0f90c3 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -35,6 +35,10 @@ class ItemLoader(object):
         self.parent = parent
         self._local_item = context['item'] = item
         self._local_values = defaultdict(list)
+        # Preprocess values if item built from dict
+        # Values need to be added to item._values if added them from dict (not with add_values)
+        for field_name, value in item.items():
+            self._values[field_name] = self._process_input_value(field_name, value)
 
     @property
     def _values(self):


Test patch:
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 8b58e4dbd..5a8ee1b2e 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -419,6 +419,43 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(item['url'], u'rabbit.hole')
         self.assertEqual(item['summary'], u'rabbithole')
 
+    def test_create_item_from_dict(self):
+        class TestItem(Item):
+            title = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+
+        input_item = {'title': 'Test item title 1'}
+        il = TestItemLoader(item=input_item)
+        # Getting output value mustn't remove value from item
+        self.assertEqual(il.load_item(), {
+            'title': 'Test item title 1',
+        })
+        self.assertEqual(il.get_output_value('title'), 'Test item title 1')
+        self.assertEqual(il.load_item(), {
+            'title': 'Test item title 1',
+        })
+
+        input_item = {'title': 'Test item title 2'}
+        il = TestItemLoader(item=input_item)
+        # Values from dict must be added to item _values
+        self.assertEqual(il._values.get('title'), 'Test item title 2')
+
+        input_item = {'title': [u'Test item title 3', u'Test item 4']}
+        il = TestItemLoader(item=input_item)
+        # Same rules must work for lists
+        self.assertEqual(il._values.get('title'),
+                         [u'Test item title 3', u'Test item 4'])
+        self.assertEqual(il.load_item(), {
+            'title': [u'Test item title 3', u'Test item 4'],
+        })
+        self.assertEqual(il.get_output_value('title'),
+                         [u'Test item title 3', u'Test item 4'])
+        self.assertEqual(il.load_item(), {
+            'title': [u'Test item title 3', u'Test item 4'],
+        })
+
 
 class ProcessorsTest(unittest.TestCase):
Repository: scrapy/scrapy
Pull request #4094 (2019-10-22 13:50:04)

Improve URL schema validation on scrapy.Request initialization
This PR should fix issue #2552 by improving the request schema validation on its initialization. 
The implementation is based on feedback posted by @pawelmhm and @hsumerf 


Patch:
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index d09eaf849..76a428199 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -66,7 +66,7 @@ class Request(object_ref):
         s = safe_url_string(url, self.encoding)
         self._url = escape_ajax(s)
 
-        if ':' not in self._url:
+        if ('://' not in self._url) and (not self._url.startswith('data:')):
             raise ValueError('Missing scheme in request url: %s' % self._url)
 
     url = property(_get_url, obsolete_setter(_set_url, 'url'))


Test patch:
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 16d7a1cb8..64f1184c3 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -52,6 +52,8 @@ class RequestTest(unittest.TestCase):
 
     def test_url_no_scheme(self):
         self.assertRaises(ValueError, self.request_class, 'foo')
+        self.assertRaises(ValueError, self.request_class, '/foo/')
+        self.assertRaises(ValueError, self.request_class, '/foo:bar')
 
     def test_headers(self):
         # Different ways of setting headers attribute
Repository: scrapy/scrapy
Pull request #4123 (2019-11-04 13:42:41)

Fix LocalCache limit issue, add tests
Taken from https://github.com/scrapy/scrapy/pull/3869/files#r342049190

This piece of code can fail with `TypeError: '>=' not supported between instances of 'int' and 'NoneType'`

I think this patch is smaller and easier than #3869, it can be merged earlier.


Patch:
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index df2b99c28..f7e3240c1 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -315,8 +315,9 @@ class LocalCache(collections.OrderedDict):
         self.limit = limit
 
     def __setitem__(self, key, value):
-        while len(self) >= self.limit:
-            self.popitem(last=False)
+        if self.limit:
+            while len(self) >= self.limit:
+                self.popitem(last=False)
         super(LocalCache, self).__setitem__(key, value)
 
 


Test patch:
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 535095b8d..7e671f627 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -7,7 +7,7 @@ if six.PY2:
 else:
     from collections.abc import Mapping, MutableMapping
 
-from scrapy.utils.datatypes import CaselessDict, SequenceExclude
+from scrapy.utils.datatypes import CaselessDict, LocalCache, SequenceExclude
 
 
 __doctests__ = ['scrapy.utils.datatypes']
@@ -242,6 +242,31 @@ class SequenceExcludeTest(unittest.TestCase):
         for v in [-3, "test", 1.1]:
             self.assertNotIn(v, d)
 
+
+class LocalCacheTest(unittest.TestCase):
+
+    def test_cache_with_limit(self):
+        cache = LocalCache(limit=2)
+        cache['a'] = 1
+        cache['b'] = 2
+        cache['c'] = 3
+        self.assertEqual(len(cache), 2)
+        self.assertNotIn('a', cache)
+        self.assertIn('b', cache)
+        self.assertIn('c', cache)
+        self.assertEqual(cache['b'], 2)
+        self.assertEqual(cache['c'], 3)
+
+    def test_cache_without_limit(self):
+        maximum = 10**4
+        cache = LocalCache()
+        for x in range(maximum):
+            cache[str(x)] = x
+        self.assertEqual(len(cache), maximum)
+        for x in range(maximum):
+            self.assertIn(str(x), cache)
+            self.assertEqual(cache[str(x)], x)
+
+
 if __name__ == "__main__":
     unittest.main()
-