Repository: LMFDB/lmfdb

Issue:
reload_all does not work with non-default separator
PR #3583 fixed the problem when using a default separator, but with a specified separator reload_all no longer works correctly (it did work prior to #3572).



Discussion:
@AndrewVSutherland, the PR #3588 should address this issue. 
Could you try again the copy_to or reload_all using  ":" as a separator?
I think there is still an issue (reload_all hit a problem on counts):

> `sage: db.reload_all("/scratch/ar_Dec29",adjust_schema=True,sep=":")
Reloading artin_field_data, artin_reps
Reloading artin_field_data...
	Loaded data into artin_field_data in 94.432 secs from /scratch/ar_Dec29/artin_field_data.txt
---------------------------------------------------------------------------

> InvalidTextRepresentation                 Traceback (most recent call last)
> <ipython-input-4-002fae6d22f4> in <module>()
> ----> 1 db.reload_all("/scratch/ar_Dec29",adjust_schema=True,sep=":")
> 
> /home/drew/github/lmfdb/lmfdb/backend/database.py in reload_all(self, data_folder, halt_on_errors, resort, reindex, restat, adjust_schema, commit, **kwds)
>    6825             for table, filedata, included in file_list:
>    6826                 try:
> -> 6827                     table.reload(*filedata, resort=resort, reindex=reindex, restat=restat, final_swap=False, silence_meta=True, adjust_schema=adjust_schema, **kwds)
>    6828                 except DatabaseError:
>    6829                     if halt_on_errors or non_existent_tables:
> 
> /home/drew/github/lmfdb/lmfdb/backend/database.py in reload(self, searchfile, extrafile, countsfile, statsfile, indexesfile, constraintsfile, metafile, resort, reindex, restat, final_swap, silence_meta, adjust_schema, commit, log_change, **kwds)
>    3714                 else:
>    3715                     self._clone(table, tmp_table)
> -> 3716                 addid, counts[table] = self._copy_from(filename, tmp_table, cols, header, kwds)
>    3717                 # Raise error if exactly one of search and extra contains ids
>    3718                 if header:
> 
> /home/drew/github/lmfdb/lmfdb/backend/database.py in _copy_from(self, filename, table, columns, header, kwds)
>     903 
>     904                 cur = self._db.cursor()
> --> 905                 cur.copy_from(F, table, columns=columns, sep=sep, **kwds)
>     906 
>     907                 if addid:
> 
> InvalidTextRepresentation: invalid input syntax for type json
> DETAIL:  Expected end of input, but found ":".
> CONTEXT:  JSON data, line 1: []:...
> COPY artin_field_data_counts_tmp, line 1, column cols: "[]:[]:527642:f:f"

`
I will try to look into this tomorrow night.

On Sun, 29 Dec 2019 at 20:01, Andrew Sutherland <notifications@github.com>
wrote:

> I think there is still an issue (reload_all hit a problem on counts):
>
> `sage: db.reload_all("/scratch/ar_Dec29",adjust_schema=True,sep=":")
> Reloading artin_field_data, artin_reps
> Reloading artin_field_data...
> Loaded data into artin_field_data in 94.432 secs from
> /scratch/ar_Dec29/artin_field_data.txt
>
> ------------------------------
>
> InvalidTextRepresentation Traceback (most recent call last)
> in ()
> ----> 1 db.reload_all("/scratch/ar_Dec29",adjust_schema=True,sep=":")
>
> /home/drew/github/lmfdb/lmfdb/backend/database.py in reload_all(self,
> data_folder, halt_on_errors, resort, reindex, restat, adjust_schema,
> commit, **kwds)
> 6825 for table, filedata, included in file_list:
> 6826 try:
> -> 6827 table.reload(*filedata, resort=resort, reindex=reindex,
> restat=restat, final_swap=False, silence_meta=True,
> adjust_schema=adjust_schema, **kwds)
> 6828 except DatabaseError:
> 6829 if halt_on_errors or non_existent_tables:
>
> /home/drew/github/lmfdb/lmfdb/backend/database.py in reload(self,
> searchfile, extrafile, countsfile, statsfile, indexesfile, constraintsfile,
> metafile, resort, reindex, restat, final_swap, silence_meta, adjust_schema,
> commit, log_change, **kwds)
> 3714 else:
> 3715 self._clone(table, tmp_table)
> -> 3716 addid, counts[table] = self._copy_from(filename, tmp_table, cols,
> header, kwds)
> 3717 # Raise error if exactly one of search and extra contains ids
> 3718 if header:
>
> /home/drew/github/lmfdb/lmfdb/backend/database.py in _copy_from(self,
> filename, table, columns, header, kwds)
> 903
> 904 cur = self._db.cursor()
> --> 905 cur.copy_from(F, table, columns=columns, sep=sep, **kwds)
> 906
> 907 if addid:
>
> InvalidTextRepresentation: invalid input syntax for type json
> DETAIL: Expected end of input, but found ":".
> CONTEXT: JSON data, line 1: []:...
> COPY artin_field_data_counts_tmp, line 1, column cols: "[]:[]:527642:f:f"
> `
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/LMFDB/lmfdb/issues/3586?email_source=notifications&email_token=AACO2BT6SA7AXGIBH7J6QIDQ3FB5DA5CNFSM4KA44D3KYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOEHZMKVQ#issuecomment-569558358>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AACO2BV7RK3RCVLNZYIYKB3Q3FB5DANCNFSM4KA44D3A>
> .
>

Now that the default separator works again (and a specified separator 
works with reload), this is not urgent.

On 2019-12-29 21:10, Edgar Costa wrote:
> I will try to look into this tomorrow night.
> 
> On Sun, 29 Dec 2019 at 20:01, Andrew Sutherland 
> <notifications@github.com>
> wrote:
> 
>> I think there is still an issue (reload_all hit a problem on counts):
>> 
>> `sage: db.reload_all("/scratch/ar_Dec29",adjust_schema=True,sep=":")
>> Reloading artin_field_data, artin_reps
>> Reloading artin_field_data...
>> Loaded data into artin_field_data in 94.432 secs from
>> /scratch/ar_Dec29/artin_field_data.txt
>> 
>> ------------------------------
>> 
>> InvalidTextRepresentation Traceback (most recent call last)
>> in ()
>> ----> 1 db.reload_all("/scratch/ar_Dec29",adjust_schema=True,sep=":")
>> 
>> /home/drew/github/lmfdb/lmfdb/backend/database.py in reload_all(self,
>> data_folder, halt_on_errors, resort, reindex, restat, adjust_schema,
>> commit, **kwds)
>> 6825 for table, filedata, included in file_list:
>> 6826 try:
>> -> 6827 table.reload(*filedata, resort=resort, reindex=reindex,
>> restat=restat, final_swap=False, silence_meta=True,
>> adjust_schema=adjust_schema, **kwds)
>> 6828 except DatabaseError:
>> 6829 if halt_on_errors or non_existent_tables:
>> 
>> /home/drew/github/lmfdb/lmfdb/backend/database.py in reload(self,
>> searchfile, extrafile, countsfile, statsfile, indexesfile, 
>> constraintsfile,
>> metafile, resort, reindex, restat, final_swap, silence_meta, 
>> adjust_schema,
>> commit, log_change, **kwds)
>> 3714 else:
>> 3715 self._clone(table, tmp_table)
>> -> 3716 addid, counts[table] = self._copy_from(filename, tmp_table, 
>> cols,
>> header, kwds)
>> 3717 # Raise error if exactly one of search and extra contains ids
>> 3718 if header:
>> 
>> /home/drew/github/lmfdb/lmfdb/backend/database.py in _copy_from(self,
>> filename, table, columns, header, kwds)
>> 903
>> 904 cur = self._db.cursor()
>> --> 905 cur.copy_from(F, table, columns=columns, sep=sep, **kwds)
>> 906
>> 907 if addid:
>> 
>> InvalidTextRepresentation: invalid input syntax for type json
>> DETAIL: Expected end of input, but found ":".
>> CONTEXT: JSON data, line 1: []:...
>> COPY artin_field_data_counts_tmp, line 1, column cols: 
>> "[]:[]:527642:f:f"
>> `
>> 
>> —
>> You are receiving this because you commented.
>> Reply to this email directly, view it on GitHub
>> <https://github.com/LMFDB/lmfdb/issues/3586?email_source=notifications&email_token=AACO2BT6SA7AXGIBH7J6QIDQ3FB5DA5CNFSM4KA44D3KYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOEHZMKVQ#issuecomment-569558358>,
>> or unsubscribe
>> <https://github.com/notifications/unsubscribe-auth/AACO2BV7RK3RCVLNZYIYKB3Q3FB5DANCNFSM4KA44D3A>
>> .
>> 


Resolving patch:
diff --git a/lmfdb/backend/database.py b/lmfdb/backend/database.py
--- a/lmfdb/backend/database.py
+++ b/lmfdb/backend/database.py
@@ -876,6 +876,7 @@ def _copy_from(self, filename, table, columns, header, kwds):
             This should be True for search and extra tables, False for counts and stats.
         - ``kwds`` -- passed on to psycopg2's copy_from
         """
+        kwds = dict(kwds) # to not modify the dict kwds, with the pop
         sep = kwds.pop("sep", u"|")
 
         with DelayCommit(self, silence=True):
@@ -1143,12 +1144,12 @@ def _reload_meta(self, meta_name, filename, search_table, sep="|"):
 
 
         with open(filename, "r") as F:
-            lines = [line for line in csv.reader(F, delimiter = "|")]
+            lines = [line for line in csv.reader(F, delimiter=sep)]
             if len(lines) == 0:
                 return
             for line in lines:
                 if line[table_name_idx] != search_table:
-                    raise RuntimeError("column %d in the file doesn't match the search table name" % table_name_idx)
+                    raise RuntimeError("in %s column %d (= %s) in the file doesn't match the search table name %s" % (filename, table_name_idx, line[table_name_idx], search_table))
 
 
         with DelayCommit(self, silence=True):
@@ -6724,6 +6725,7 @@ def reload_all(self, data_folder, halt_on_errors=True, resort=None, reindex=True
         if not os.path.isdir(data_folder):
             raise ValueError(
                     "The path {} is not a directory".format(data_folder))
+        sep = kwds.get("sep", u"|")
         with DelayCommit(self, commit, silence=True):
             file_list = []
             tablenames = []
@@ -6750,7 +6752,7 @@ def reload_all(self, data_folder, halt_on_errors=True, resort=None, reindex=True
                     # read metafile
                     rows = []
                     with open(metafile, "r") as F:
-                        rows = [line for line in csv.reader(F, delimiter = "|")]
+                        rows = [line for line in csv.reader(F, delimiter=sep)]
                     if len(rows) != 1:
                         raise RuntimeError("Expected only one row in {0}")
                     meta = dict(zip(_meta_tables_cols, rows[0]))
@@ -6838,7 +6840,7 @@ def reload_all(self, data_folder, halt_on_errors=True, resort=None, reindex=True
             for table, filedata, included in file_list:
                 if table in failures:
                     continue
-                table.reload_final_swap(tables=included, metafile=filedata[-1])
+                table.reload_final_swap(tables=included, metafile=filedata[-1], sep=sep)
 
         if failures:
             print("Reloaded %s"%(", ".join(tablenames)))


Test patch: