nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: scrapy/scrapy
Pull request #1725 (2016-01-26 14:40:25)

[MRG+1] Disable SpiderState extension if no JOBDIR set


Patch:
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 3799c7c66..2220cbd8f 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -2,6 +2,7 @@ import os
 from six.moves import cPickle as pickle
 
 from scrapy import signals
+from scrapy.exceptions import NotConfigured
 from scrapy.utils.job import job_dir
 
 class SpiderState(object):
@@ -12,7 +13,11 @@ class SpiderState(object):
 
     @classmethod
     def from_crawler(cls, crawler):
-        obj = cls(job_dir(crawler.settings))
+        jobdir = job_dir(crawler.settings)
+        if not jobdir:
+            raise NotConfigured
+
+        obj = cls(jobdir)
         crawler.signals.connect(obj.spider_closed, signal=signals.spider_closed)
         crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened)
         return obj


Test patch:
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index d83015bd9..d1d6debec 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -4,6 +4,8 @@ from twisted.trial import unittest
 
 from scrapy.extensions.spiderstate import SpiderState
 from scrapy.spiders import Spider
+from scrapy.exceptions import NotConfigured
+from scrapy.utils.test import get_crawler
 
 
 class SpiderStateTest(unittest.TestCase):
@@ -34,3 +36,7 @@ class SpiderStateTest(unittest.TestCase):
         ss.spider_opened(spider)
         self.assertEqual(spider.state, {})
         ss.spider_closed(spider)
+
+    def test_not_configured(self):
+        crawler = get_crawler(Spider)
+        self.assertRaises(NotConfigured, SpiderState.from_crawler, crawler)
Repository: scrapy/scrapy
Pull request #1737 (2016-01-27 17:49:56)

[MRG+1] Fix PythonItemExporter and CSVExporter for non-string item types
This PR fixes the error related to PythonItemExporter with non-string types reported here: https://github.com/scrapy/scrapy/commit/c76190d491fca9f35b6758bdc06c34d77f5d9be9#commitcomment-15693798.

It also fixes how CSVItemExporter represents non-string datatypes in CSV.

Instead of generating this (for a datetime object):
   ```2,True,False,2015-01-01 01:01:01```

It was generating this:
    ```True,2,False,"datetime.datetime(2015, 1, 1, 1, 1, 1)"```



Patch:
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 145468dbe..35f50838b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -200,7 +200,7 @@ class CsvItemExporter(BaseItemExporter):
             try:
                 yield to_native_str(s)
             except TypeError:
-                yield to_native_str(repr(s))
+                yield s
 
     def _write_headers_and_set_fields_to_export(self, item):
         if self.include_headers_line:
@@ -273,10 +273,10 @@ class PythonItemExporter(BaseItemExporter):
             return dict(self._serialize_dict(value))
         if is_listlike(value):
             return [self._serialize_value(v) for v in value]
-        if self.binary:
-            return to_bytes(value, encoding=self.encoding)
-        else:
-            return to_unicode(value, encoding=self.encoding)
+        encode_func = to_bytes if self.binary else to_unicode
+        if isinstance(value, (six.text_type, bytes)):
+            return encode_func(value, encoding=self.encoding)
+        return value
 
     def _serialize_dict(self, value):
         for key, val in six.iteritems(value):


Test patch:
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 1633e1039..97c09a495 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -134,6 +134,19 @@ class PythonItemExporterTest(BaseItemExporterTest):
         expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
         self.assertEqual(expected, exporter.export_item(value))
 
+    def test_other_python_types_item(self):
+        from datetime import datetime
+        now = datetime.now()
+        item = {
+            'boolean': False,
+            'number': 22,
+            'time': now,
+            'float': 3.14,
+        }
+        ie = self._get_exporter()
+        exported = ie.export_item(item)
+        self.assertEqual(exported, item)
+
 
 class PprintItemExporterTest(BaseItemExporterTest):
 
@@ -258,6 +271,21 @@ class CsvItemExporterTest(BaseItemExporterTest):
             expected='"[4, 8]",John\r\n',
         )
 
+    def test_other_python_types_item(self):
+        from datetime import datetime
+        now = datetime(2015, 1, 1, 1, 1, 1)
+        item = {
+            'boolean': False,
+            'number': 22,
+            'time': now,
+            'float': 3.14,
+        }
+        self.assertExportResult(
+            item=item,
+            include_headers_line=False,
+            expected='22,False,3.14,2015-01-01 01:01:01\r\n'
+        )
+
 
 class XmlItemExporterTest(BaseItemExporterTest):
Repository: scrapy/scrapy
Pull request #1747 (2016-01-29 21:29:40)

[MRG+1] Fix bug on XMLItemExporter with non-string fields in items
This PR fixes https://github.com/scrapy/scrapy/issues/1738

codecov-io: ## [Current coverage][1] is `83.33%`
> Merging **#1747** into **master** will increase coverage by **+0.01%** as of [`cf5c235`][3]



[1]: https://codecov.io/github/scrapy/scrapy?ref=cf5c2355959309807b2f5b7e971a918dbb9f555f
[2]: https://codecov.io/github/scrapy/scrapy/features/suggestions?ref=cf5c2355959309807b2f5b7e971a918dbb9f555f
[3]: https://codecov.io/github/scrapy/scrapy/commit/cf5c2355959309807b2f5b7e971a918dbb9f555f
[4]: https://codecov.io/github/scrapy/scrapy/compare/a35aec71e96b0c0288c370afa425e8e700dca8b3...cf5c2355959309807b2f5b7e971a918dbb9f555f

> Powered by [Codecov](https://codecov.io). Updated on successful CI builds.

Patch:
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 35f50838b..360007c0f 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -144,8 +144,10 @@ class XmlItemExporter(BaseItemExporter):
         elif is_listlike(serialized_value):
             for value in serialized_value:
                 self._export_xml_field('value', value)
-        else:
+        elif isinstance(serialized_value, six.text_type):
             self._xg_characters(serialized_value)
+        else:
+            self._xg_characters(str(serialized_value))
         self.xg.endElement(name)
 
     # Workaround for http://bugs.python.org/issue17606


Test patch:
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 7ba5a0af6..cd72c661a 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -376,6 +376,20 @@ class XmlItemExporterTest(BaseItemExporterTest):
             b'</items>'
         )
 
+    def test_nonstring_types_item(self):
+        item = self._get_nonstring_types_item()
+        self.assertExportResult(item,
+            b'<?xml version="1.0" encoding="utf-8"?>\n'
+            b'<items>'
+               b'<item>'
+                   b'<float>3.14</float>'
+                   b'<boolean>False</boolean>'
+                   b'<number>22</number>'
+                   b'<time>2015-01-01 01:01:01</time>'
+               b'</item>'
+            b'</items>'
+        )
+
 
 class JsonLinesItemExporterTest(BaseItemExporterTest):