Repository: ckan/ckan

Issue:
CKAN is incorrectly adding Z timezone indicator to datetime being passed to solr via search/index.py
 CKAN version: 2.9.9

If a package datetime field is fully specified to ISO-8601 (eg collection_date=2019-10-10T01:15:00Z), on processing through the index_package method, the resulting datetime string is invalid, and unable to be parsed by solr. 

### Steps to reproduce
In a solr-enabled environment
Have a field on a  ckan schema ending in _date
Using the API call create a package with a value of "2023-01-11" into the date field. - THIS WILL WORK.
Using the API call, patch the package using a value for the date field of "2019-10-10T01:15:00Z" This will fail, with this error:

pysolr.SolrError: Solr responded with an error (HTTP 400): [Reason: ERROR: [doc=535287a506ddae367b9fce761adb18fb] Error adding field 'collection_date'='2019-10-04T00:22:00+00:00Z' msg=Invalid Date in Date Math String:'2019-10-04T00:22:00+00:00Z']

The problem is the "double" timezone specifier: +00:00Z  - it should be EITHER +00:00 OR Z, not both.

### Expected behavior

If the datetime is fully specfied, that fully specfied datetime should be passed into solr, and stored correctly.

### Additional details

This is the problem line of code. 
https://github.com/ckan/ckan/blob/fd88d1f4c52c8ee247883549ca23500693e2e2a4/ckan/lib/search/index.py#LL250C1-L250C55

The parse(value, default=bogus_date) - Line 248 statement parses the date, and the parse function adds the +00:00 tiemzone information.
Adding the Z to the date.isoformat() - line  250 makes the timezone information incorrect. If Z is required when a timezone is NOT specified, then logic needs to be added to NOT add the Z if the timezone was specified and picked up by the parse.

Here is the stacktrace:
solr_1         | 2023-06-20 05:35:01.253 INFO  (qtp1581078471-23) [   x:ckan] o.a.s.c.S.Request [ckan]  webapp=/solr path=/select params={q=name:"bpa-amdb-genomics-amplicon-16s-137726-j6562_aaggagcg-gtctagtg"+OR+id:"bpa-amdb-genomics-amplicon-16s-137726-j6562_aaggagcg-gtctagtg"&fq=site_id:"default"+%2Bentity_type:package&rows=1&wt=json} hits=1 status=0 QTime=0
solr_1         | 2023-06-20 05:35:01.820 INFO  (qtp1581078471-15) [   x:ckan] o.a.s.u.p.LogUpdateProcessorFactory [ckan]  webapp=/solr path=/update params={commit=true}{} 0 198
ckan_1         | 2023-06-20 05:35:01,843 ERROR [ckan.lib.search] Solr returned an error: Solr responded with an error (HTTP 400): [Reason: ERROR: [doc=535287a506ddae367b9fce761adb18fb] Error adding field 'collection_date'='2019-10-04T00:22:00+00:00Z' msg=Invalid Date in Date Math String:'2019-10-04T00:22:00+00:00Z']
ckan_1         | Traceback (most recent call last):
ckan_1         |   File "/app/ckan/ckan/lib/search/index.py", line 304, in index_package
ckan_1         |     conn.add(docs=[pkg_dict], commit=commit)
ckan_1         |   File "/env/lib/python3.9/site-packages/pysolr.py", line 890, in add
ckan_1         |     return self._update(m, commit=commit, softCommit=softCommit, waitFlush=waitFlush, waitSearcher=waitSearcher,
ckan_1         |   File "/env/lib/python3.9/site-packages/pysolr.py", line 478, in _update
ckan_1         |     return self._send_request('post', path, message, {'Content-type': 'text/xml; charset=utf-8'})
ckan_1         |   File "/env/lib/python3.9/site-packages/pysolr.py", line 393, in _send_request
ckan_1         |     raise SolrError(error_message % (resp.status_code, solr_message))
ckan_1         | pysolr.SolrError: Solr responded with an error (HTTP 400): [Reason: ERROR: [doc=535287a506ddae367b9fce761adb18fb] Error adding field 'collection_date'='2019-10-04T00:22:00+00:00Z' msg=Invalid Date in Date Math String:'2019-10-04T00:22:00+00:00Z']



Discussion:

Resolving patch:
diff --git a/ckan/lib/search/index.py b/ckan/lib/search/index.py
--- a/ckan/lib/search/index.py
+++ b/ckan/lib/search/index.py
@@ -6,9 +6,8 @@
 import logging
 import collections
 import json
-import datetime
 import re
-from dateutil.parser import parse
+from dateutil.parser import parse, ParserError as DateParserError
 from typing import Any, NoReturn, Optional
 
 import six
@@ -235,27 +234,22 @@ def index_package(self,
         pkg_dict['dataset_type'] = pkg_dict['type']
 
         # clean the dict fixing keys and dates
-        # FIXME where are we getting these dirty keys from?  can we not just
-        # fix them in the correct place or is this something that always will
-        # be needed?  For my data not changing the keys seems to not cause a
-        # problem.
         new_dict = {}
-        bogus_date = datetime.datetime(1, 1, 1)
         for key, value in pkg_dict.items():
             key = six.ensure_str(key)
             if key.endswith('_date'):
+                if not value:
+                    continue
                 try:
-                    date = parse(value, default=bogus_date)
-                    if date != bogus_date:
-                        value = date.isoformat() + 'Z'
-                    else:
-                        # The date field was empty, so dateutil filled it with
-                        # the default bogus date
-                        value = None
-                except (IndexError, TypeError, ValueError):
-                    log.error('%r: %r value of %r is not a valid date', pkg_dict['id'], key, value)
+                    date = parse(value)
+                    value = date.isoformat()
+                    if not date.tzinfo:
+                        value += 'Z'
+                except DateParserError:
+                    log.warning('%r: %r value of %r is not a valid date', pkg_dict['id'], key, value)
                     continue
             new_dict[key] = value
+
         pkg_dict = new_dict
 
         for k in ('title', 'notes', 'title_string'):


Test patch:
diff --git a/ckan/tests/lib/search/test_index.py b/ckan/tests/lib/search/test_index.py
--- a/ckan/tests/lib/search/test_index.py
+++ b/ckan/tests/lib/search/test_index.py
@@ -124,6 +124,7 @@ def test_index_date_field(self):
                 "extras": [
                     {"key": "test_date", "value": "2014-03-22"},
                     {"key": "test_tim_date", "value": "2014-03-22 05:42:14"},
+                    {"key": "test_full_iso_date", "value": "2019-10-10T01:15:00Z"},
                 ]
             }
         )
@@ -142,6 +143,31 @@ def test_index_date_field(self):
             response.docs[0]["test_tim_date"].strftime("%Y-%m-%d %H:%M:%S")
             == "2014-03-22 05:42:14"
         )
+        assert (
+            response.docs[0]["test_full_iso_date"].strftime("%Y-%m-%d %H:%M:%S")
+            == "2019-10-10 01:15:00"
+        )
+
+    def test_index_date_empty_value(self):
+
+        pkg_dict = self.base_package_dict.copy()
+        pkg_dict.update(
+            {
+                "extras": [
+                    {"key": "test_empty_date", "value": ""},
+                    {"key": "test_none_date", "value": None},
+                ]
+            }
+        )
+
+        self.package_index.index_package(pkg_dict)
+
+        response = self.solr_client.search(q="name:monkey", fq=self.fq)
+
+        assert len(response) == 1
+
+        assert "test_empty_date" not in response.docs[0]
+        assert "test_none_date" not in response.docs[0]
 
     def test_index_date_field_wrong_value(self):