Repository: RDFLib/rdflib

Issue:
csv2rdf uses Python 2 features broken by Python 3
I have tried to run **csv2rdf** under both Windows 10 and a(n Alpine) Linux Docker container and it failed both times, due to differences between Python 2 and Python 3 strings and bytestreams.

I have been able to patch the module using the following three **sed** commands:

```sh
# Replace .next() with next(...)
sed -i 's/header_labels = list(csvreader.next())/header_labels = list(next(csvreader))/g' /usr/local/lib/python3.8/site-packages/rdflib/tools/csv2rdf.py

# Python 3 strings are natively Unicode
sed -i 's/yield \[str(cell, "utf-8", errors="replace") for cell in row\]/yield \[cell for cell in row\]/g' /usr/local/lib/python3.8/site-packages/rdflib/tools/csv2rdf.py

# Reference https://stackoverflow.com/questions/4374455/how-to-set-sys-stdout-encoding-in-python-3
sed -i 's/self.OUT = codecs.getwriter("utf-8")(sys.stdout, errors="replace")/self.OUT = codecs.getwriter("utf-8")(sys.stdout.detach(), errors="replace")/g' /usr/local/lib/python3.8/site-packages/rdflib/tools/csv2rdf.py
```

but this seems to alter some of **csv2rdf**'s incidental behaviour, such as showing the help text and printing extra output about the buffer being detatched.

It would be good if someone who knew what they were doing could take a look at this and update this tool for Python 3.

Discussion:

Patch:
diff --git a/rdflib/tools/csv2rdf.py b/rdflib/tools/csv2rdf.py
index 812ffadc..64b7c6eb 100644
--- a/rdflib/tools/csv2rdf.py
+++ b/rdflib/tools/csv2rdf.py
@@ -126,8 +126,7 @@ def csv_reader(csv_data, dialect=csv.excel, **kwargs):
 
     csv_reader = csv.reader(csv_data, dialect=dialect, **kwargs)
     for row in csv_reader:
-        # decode UTF-8 back to Unicode, cell by cell:
-        yield [str(cell, "utf-8", errors="replace") for cell in row]
+        yield row
 
 
 def prefixuri(x, prefix, class_=None):
@@ -315,7 +314,7 @@ class CSV2RDF(object):
         self.COLUMNS = {}
         self.PROPS = {}
 
-        self.OUT = codecs.getwriter("utf-8")(sys.stdout, errors="replace")
+        self.OUT = sys.stdout
 
         self.triples = 0
 
@@ -346,7 +345,7 @@ class CSV2RDF(object):
             next(csvreader)
 
         # read header line
-        header_labels = list(csvreader.next())
+        header_labels = list(next(csvreader))
         headers = dict(enumerate([self.PROPBASE[toProperty(x)] for x in header_labels]))
         # override header properties if some are given
         for k, v in self.PROPS.items():


Test patch:
diff --git a/test/csv/realestate.csv b/test/csv/realestate.csv
new file mode 100644
index 00000000..e6614ff7
--- /dev/null
+++ b/test/csv/realestate.csv
@@ -0,0 +1,20 @@
+street,city,zip,state,beds,baths,sq__ft,type,sale_date,price,latitude,longitude
+3526 HIGH ST,SACRAMENTO,95838,CA,2,1,836,Residential,Wed May 21 00:00:00 EDT 2008,59222,38.631913,-121.434879
+51 OMAHA CT,SACRAMENTO,95823,CA,3,1,1167,Residential,Wed May 21 00:00:00 EDT 2008,68212,38.478902,-121.431028
+2796 BRANCH ST,SACRAMENTO,95815,CA,2,1,796,Residential,Wed May 21 00:00:00 EDT 2008,68880,38.618305,-121.443839
+2805 JANETTE WAY,SACRAMENTO,95815,CA,2,1,852,Residential,Wed May 21 00:00:00 EDT 2008,69307,38.616835,-121.439146
+6001 MCMAHON DR,SACRAMENTO,95824,CA,2,1,797,Residential,Wed May 21 00:00:00 EDT 2008,81900,38.51947,-121.435768
+5828 PEPPERMILL CT,SACRAMENTO,95841,CA,3,1,1122,Condo,Wed May 21 00:00:00 EDT 2008,89921,38.662595,-121.327813
+6048 OGDEN NASH WAY,SACRAMENTO,95842,CA,3,2,1104,Residential,Wed May 21 00:00:00 EDT 2008,90895,38.681659,-121.351705
+2561 19TH AVE,SACRAMENTO,95820,CA,3,1,1177,Residential,Wed May 21 00:00:00 EDT 2008,91002,38.535092,-121.481367
+11150 TRINITY RIVER DR Unit 114,RANCHO CORDOVA,95670,CA,2,2,941,Condo,Wed May 21 00:00:00 EDT 2008,94905,38.621188,-121.270555
+7325 10TH ST,RIO LINDA,95673,CA,3,2,1146,Residential,Wed May 21 00:00:00 EDT 2008,98937,38.700909,-121.442979
+645 MORRISON AVE,SACRAMENTO,95838,CA,3,2,909,Residential,Wed May 21 00:00:00 EDT 2008,100309,38.637663,-121.45152
+4085 FAWN CIR,SACRAMENTO,95823,CA,3,2,1289,Residential,Wed May 21 00:00:00 EDT 2008,106250,38.470746,-121.458918
+2930 LA ROSA RD,SACRAMENTO,95815,CA,1,1,871,Residential,Wed May 21 00:00:00 EDT 2008,106852,38.618698,-121.435833
+2113 KIRK WAY,SACRAMENTO,95822,CA,3,1,1020,Residential,Wed May 21 00:00:00 EDT 2008,107502,38.482215,-121.492603
+4533 LOCH HAVEN WAY,SACRAMENTO,95842,CA,2,2,1022,Residential,Wed May 21 00:00:00 EDT 2008,108750,38.672914,-121.35934
+7340 HAMDEN PL,SACRAMENTO,95842,CA,2,2,1134,Condo,Wed May 21 00:00:00 EDT 2008,110700,38.700051,-121.351278
+6715 6TH ST,RIO LINDA,95673,CA,2,1,844,Residential,Wed May 21 00:00:00 EDT 2008,113263,38.689591,-121.452239
+6236 LONGFORD DR Unit 1,CITRUS HEIGHTS,95621,CA,2,1,795,Condo,Wed May 21 00:00:00 EDT 2008,116250,38.679776,-121.314089
+250 PERALTA AVE,SACRAMENTO,95833,CA,2,1,588,Residential,Wed May 21 00:00:00 EDT 2008,120000,38.612099,-121.469095
diff --git a/test/test_csv2rdf.py b/test/test_csv2rdf.py
new file mode 100644
index 00000000..d0bba696
--- /dev/null
+++ b/test/test_csv2rdf.py
@@ -0,0 +1,27 @@
+import subprocess
+import unittest
+from os import remove
+from tempfile import mkstemp
+
+
+class CSV2RDFTest(unittest.TestCase):
+    def test_csv2rdf_cli(self):
+        completed = subprocess.run(
+            ["csv2rdf", "test/csv/realestate.csv"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        )
+        self.assertEqual(completed.returncode, 0)
+        self.assertRegex(completed.stderr, r"Converted \d+ rows into \d+ triples.")
+        self.assertRegex(completed.stderr, r"Took [\d\.]+ seconds.")
+        self.assertIn("<http://example.org/instances/0>", completed.stdout)
+        self.assertIn("<http://example.org/props/zip>", completed.stdout)
+
+    def test_csv2rdf_cli_fileout(self):
+        _, fname = mkstemp()
+        completed = subprocess.run(["csv2rdf", "-o", fname, "test/csv/realestate.csv"],)
+        self.assertEqual(completed.returncode, 0)
+        with open(fname) as f:
+            self.assertGreater(len(f.readlines()), 0)
+        remove(fname)