Repository: ESMCI/cime

Issue:
use of python datetime in case_st_archive precludes dates beyond 9999
We don't need to use datetime here.  


Discussion:
Would it make sense to just write our own datetime object?
Writing a fully general datetime is pretty involved, but I think we can get the functionality we are currently using without resorting to that. 
That's fair. I'm not fond of this idea, since it seems like too much of a hack, but thinking about it, maybe we could get away with just changing MAX_DATE to something much larger? IIRC, the Python code only uses it in one place, and raises an exception if we're too large, though it may cause issues with formatting.
I tried that, couldn't figure out how to change that value - maybe you can?   I was also worried about side-effects.
I'm finally starting on this, but was wondering what the format for dates with years >9999 was?
The format most dates use has only 4 digits for the year.

Unsurprising answer: `hugedate.ne4np4_oQU240.cime.dice.r.10000-01-05-00000.nc`
Also note that CLM4 doesn't support years>9999 either, or I need to update it

Resolving patch:
diff --git a/scripts/lib/CIME/SystemTests/ldsta.py b/scripts/lib/CIME/SystemTests/ldsta.py
--- a/scripts/lib/CIME/SystemTests/ldsta.py
+++ b/scripts/lib/CIME/SystemTests/ldsta.py
@@ -17,6 +17,15 @@
 
 logger = logging.getLogger(__name__)
 
+# datetime objects can't be used anywhere else
+def _date_to_datetime(date_obj):
+    return datetime.datetime(year = date_obj.year(),
+                             month = date_obj.month(),
+                             day = date_obj.day(),
+                             hour = date_obj.hour(),
+                             minute = date_obj.minute(),
+                             second = date_obj.second())
+
 class LDSTA(SystemTestsCommon):
 
     def __init__(self, case):
@@ -31,7 +40,7 @@ def run_phase(self):
             shutil.rmtree(archive_dir)
         self.run_indv()
         # finished running, so all archive files should exist
-        start_date = get_file_date(self._case.get_value('RUN_STARTDATE'))
+        start_date = _date_to_datetime(get_file_date(self._case.get_value('RUN_STARTDATE')))
         rest_dir = os.path.join(archive_dir, 'rest')
         delta_day = datetime.timedelta(1)
         current_date = start_date + delta_day
@@ -44,7 +53,7 @@ def run_phase(self):
                                                           current_date.month,
                                                           current_date.day)
             case_st_archive(self._case, last_date_str=current_date_str, copy_only=False)
-            archive_dates = [get_file_date(fname)
+            archive_dates = [_date_to_datetime(get_file_date(fname))
                              for fname in glob.glob(os.path.join(rest_dir, '*'))]
             while next_datecheck <= current_date:
                 expect(next_datecheck in archive_dates,
diff --git a/scripts/lib/CIME/case_st_archive.py b/scripts/lib/CIME/case_st_archive.py
--- a/scripts/lib/CIME/case_st_archive.py
+++ b/scripts/lib/CIME/case_st_archive.py
@@ -7,8 +7,8 @@
 from CIME.XML.standard_module_setup import *
 from CIME.case_submit               import submit
 from CIME.utils                     import run_and_log_case_status, ls_sorted_by_mtime, symlink_force
+from CIME.date                      import date
 from os.path                        import isdir, join
-import datetime
 
 logger = logging.getLogger(__name__)
 
@@ -24,7 +24,7 @@ def _get_archive_file_fn(copy_only):
 def _get_datenames(case):
 ###############################################################################
     """
-    Returns the datetime objects specifying the times of each file
+    Returns the date objects specifying the times of each file
     Note we are assuming that the coupler restart files exist and are consistent with other component datenames
     Not doc-testable due to filesystem dependence
     """
@@ -49,15 +49,15 @@ def _get_datenames(case):
 
     datenames = []
     for filename in files:
-        date = get_file_date(filename)
-        datenames.append(date)
+        file_date = get_file_date(filename)
+        datenames.append(file_date)
     return datenames
 
 ###############################################################################
 def get_file_date(filename):
 ###############################################################################
     """
-    Returns the date associated with the filename as a datetime object representing the correct date
+    Returns the date associated with the filename as a date object representing the correct date
     Formats supported:
     "%Y-%m-%d_%h.%M.%s
     "%Y-%m-%d_%05s"
@@ -67,29 +67,29 @@ def get_file_date(filename):
     "%Y.%m"
 
     >>> get_file_date("./ne4np4_oQU240.cam.r.0001-01-06-00435.nc")
-    datetime.datetime(1, 1, 6, 0, 7, 15)
+    date(1, 1, 6, 0, 7, 15)
     >>> get_file_date("./ne4np4_oQU240.cam.r.0010-1-06_00435.nc")
-    datetime.datetime(10, 1, 6, 0, 7, 15)
+    date(10, 1, 6, 0, 7, 15)
     >>> get_file_date("./ne4np4_oQU240.cam.r.0010-10.nc")
-    datetime.datetime(10, 10, 1, 0, 0)
+    date(10, 10, 1, 0, 0, 0)
     >>> get_file_date("0064-3-8_10.20.30.nc")
-    datetime.datetime(64, 3, 8, 10, 20, 30)
+    date(64, 3, 8, 10, 20, 30)
     >>> get_file_date("0140-3-5")
-    datetime.datetime(140, 3, 5, 0, 0)
+    date(140, 3, 5, 0, 0, 0)
     >>> get_file_date("0140-3")
-    datetime.datetime(140, 3, 1, 0, 0)
+    date(140, 3, 1, 0, 0, 0)
     >>> get_file_date("0140.3")
-    datetime.datetime(140, 3, 1, 0, 0)
+    date(140, 3, 1, 0, 0, 0)
     """
 
     #
     # TODO: Add these to config_archive.xml, instead of here
     # Note these must be in order of most specific to least
     # so that lesser specificities aren't used to parse greater ones
-    re_formats = [r"[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}_[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2}", # yyyy-mm-dd_hh.MM.ss
-                  r"[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}[\-_][0-9]{1,5}",                     # yyyy-mm-dd_sssss
-                  r"[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}",                                    # yyyy-mm-dd
-                  r"[0-9]{4}[\-\.][0-9]{1,2}",                                          # yyyy-mm
+    re_formats = [r"[0-9]*[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}_[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2}", # [yy...]yyyy-mm-dd_hh.MM.ss
+                  r"[0-9]*[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}[\-_][0-9]{1,5}",                     # [yy...]yyyy-mm-dd_sssss
+                  r"[0-9]*[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}",                                    # [yy...]yyyy-mm-dd
+                  r"[0-9]*[0-9]{4}[\-\.][0-9]{1,2}",                                          # [yy...]yyyy-mm
     ]
 
     for re_str in re_formats:
@@ -107,65 +107,50 @@ def get_file_date(filename):
             if len(date_tuple) == 4:
                 second = date_tuple[3]
             elif len(date_tuple) == 6:
-                # Create a datetime object with arbitrary year, month, day, but the correct time of day
+                # Create a date object with arbitrary year, month, day, but the correct time of day
                 # Then use _get_day_second to get the time of day in seconds
-                second = _get_day_second(datetime.datetime(1, 1, 1,
-                                                           hour = date_tuple[3],
-                                                           minute = date_tuple[4],
-                                                           second = date_tuple[5]))
-        return datetime.datetime(year, month, day) + datetime.timedelta(seconds = second)
+                second = date.hms_to_second(hour = date_tuple[3],
+                                            minute = date_tuple[4],
+                                            second = date_tuple[5])
+        return date(year, month, day, 0, 0, second)
 
     # Not a valid filename date format
     logger.debug("{} is a filename without a supported date!".format(filename))
     return None
 
-def _get_day_second(date):
-    """
-    Returns the total seconds that have elapsed since the beginning of the day
-    """
-    SECONDS_PER_HOUR = 3600
-    SECONDS_PER_MINUTE = 60
-    return (date.second
-            + date.minute * SECONDS_PER_MINUTE
-            + date.hour * SECONDS_PER_HOUR)
-
-def _datetime_str(date):
+def _datetime_str(_date):
     """
     Returns the standard format associated with filenames.
-    Note unfortunately datetime.datetime.strftime expects years > 1900
-    to support abbreviations, so we can't use that here
 
-    >>> _datetime_str(datetime.datetime(5, 8, 22))
+    >>> _datetime_str(date(5, 8, 22))
     '0005-08-22-00000'
     >>> _datetime_str(get_file_date("0011-12-09-00435"))
     '0011-12-09-00435'
     """
 
     format_string = "{year:04d}-{month:02d}-{day:02d}-{seconds:05d}"
-    return format_string.format(year = date.year,
-                                month = date.month,
-                                day = date.day,
-                                seconds = _get_day_second(date))
+    return format_string.format(year = _date.year(),
+                                month = _date.month(),
+                                day = _date.day(),
+                                seconds = _date.second_of_day())
 
-def _datetime_str_mpas(date):
+def _datetime_str_mpas(_date):
     """
     Returns the mpas format associated with filenames.
-    Note unfortunately datetime.datetime.strftime expects years > 1900
-    to support abbreviations, so we can't use that here
 
-    >>> _datetime_str_mpas(datetime.datetime(5, 8, 22))
+    >>> _datetime_str_mpas(date(5, 8, 22))
     '0005-08-22_00:00:00'
     >>> _datetime_str_mpas(get_file_date("0011-12-09-00435"))
     '0011-12-09_00:07:15'
     """
 
     format_string = "{year:04d}-{month:02d}-{day:02d}_{hours:02d}:{minutes:02d}:{seconds:02d}"
-    return format_string.format(year = date.year,
-                                month = date.month,
-                                day = date.day,
-                                hours = date.hour,
-                                minutes = date.minute,
-                                seconds = date.second)
+    return format_string.format(year = _date.year(),
+                                month = _date.month(),
+                                day = _date.day(),
+                                hours = _date.hour(),
+                                minutes = _date.minute(),
+                                seconds = _date.second())
 
 ###############################################################################
 def _get_ninst_info(case, compclass):
@@ -653,7 +638,7 @@ def case_st_archive(case, last_date_str=None, archive_incomplete_logs=True, copy
     case.load_env(job="case.st_archive")
     if last_date_str is not None:
         try:
-            last_date = datetime.datetime.strptime(last_date_str, '%Y-%m-%d')
+            last_date = get_file_date(last_date_str)
         except ValueError:
             expect(False, 'Could not parse the last date to archive')
     else:
diff --git a/scripts/lib/CIME/date.py b/scripts/lib/CIME/date.py
new file mode 100644
--- /dev/null
+++ b/scripts/lib/CIME/date.py
@@ -0,0 +1,210 @@
+
+class date:
+    """
+    Simple struct for holding dates and the time of day and performing comparisons
+
+    Difference in Hour, Minute, or Second
+    >>> date(4, 5, 6, 9) == date(4, 5, 6, 8)
+    False
+    >>> date(4, 5, 6, 9) != date(4, 5, 6, 8)
+    True
+    >>> date(4, 5, 6, 9) < date(4, 5, 6, 8)
+    False
+    >>> date(4, 5, 6, 9) <= date(4, 5, 6, 8)
+    False
+    >>> date(4, 5, 6, 9) >= date(4, 5, 6, 8)
+    True
+    >>> date(4, 5, 6, 9) > date(4, 5, 6, 8)
+    True
+
+    >>> date(4, 5, 6, 4) == date(4, 5, 6, 8)
+    False
+    >>> date(4, 5, 6, 4) != date(4, 5, 6, 8)
+    True
+    >>> date(4, 5, 6, 4) < date(4, 5, 6, 8)
+    True
+    >>> date(4, 5, 6, 4) <= date(4, 5, 6, 8)
+    True
+    >>> date(4, 5, 6, 4) >= date(4, 5, 6, 8)
+    False
+    >>> date(4, 5, 6, 4) > date(4, 5, 6, 8)
+    False
+
+    Difference in Day
+    >>> date(4, 5, 8, 8) == date(4, 5, 6, 8)
+    False
+    >>> date(4, 5, 8, 8) != date(4, 5, 6, 8)
+    True
+    >>> date(4, 5, 8, 8) < date(4, 5, 6, 8)
+    False
+    >>> date(4, 5, 8, 8) <= date(4, 5, 6, 8)
+    False
+    >>> date(4, 5, 8, 8) >= date(4, 5, 6, 8)
+    True
+    >>> date(4, 5, 8, 8) > date(4, 5, 6, 8)
+    True
+
+    >>> date(4, 5, 5, 8) == date(4, 5, 6, 8)
+    False
+    >>> date(4, 5, 5, 8) != date(4, 5, 6, 8)
+    True
+    >>> date(4, 5, 5, 8) < date(4, 5, 6, 8)
+    True
+    >>> date(4, 5, 5, 8) <= date(4, 5, 6, 8)
+    True
+    >>> date(4, 5, 5, 8) >= date(4, 5, 6, 8)
+    False
+    >>> date(4, 5, 5, 8) > date(4, 5, 6, 8)
+    False
+
+    Difference in Month
+    >>> date(4, 6, 6, 8) == date(4, 5, 6, 8)
+    False
+    >>> date(4, 6, 6, 8) != date(4, 5, 6, 8)
+    True
+    >>> date(4, 6, 6, 8) < date(4, 5, 6, 8)
+    False
+    >>> date(4, 6, 6, 8) <= date(4, 5, 6, 8)
+    False
+    >>> date(4, 6, 6, 8) >= date(4, 5, 6, 8)
+    True
+    >>> date(4, 6, 6, 8) > date(4, 5, 6, 8)
+    True
+
+    >>> date(4, 4, 6, 8) == date(4, 5, 6, 8)
+    False
+    >>> date(4, 4, 6, 8) != date(4, 5, 6, 8)
+    True
+    >>> date(4, 4, 6, 8) < date(4, 5, 6, 8)
+    True
+    >>> date(4, 4, 6, 8) <= date(4, 5, 6, 8)
+    True
+    >>> date(4, 4, 6, 8) >= date(4, 5, 6, 8)
+    False
+    >>> date(4, 4, 6, 8) > date(4, 5, 6, 8)
+    False
+
+    Difference in Year
+    >>> date(5, 5, 6, 8) == date(4, 5, 6, 8)
+    False
+    >>> date(5, 5, 6, 8) != date(4, 5, 6, 8)
+    True
+    >>> date(5, 5, 6, 8) < date(4, 5, 6, 8)
+    False
+    >>> date(5, 5, 6, 8) <= date(4, 5, 6, 8)
+    False
+    >>> date(5, 5, 6, 8) >= date(4, 5, 6, 8)
+    True
+    >>> date(5, 5, 6, 8) > date(4, 5, 6, 8)
+    True
+
+    >>> date(3, 5, 6, 8) == date(4, 5, 6, 8)
+    False
+    >>> date(3, 5, 6, 8) != date(4, 5, 6, 8)
+    True
+    >>> date(3, 5, 6, 8) < date(4, 5, 6, 8)
+    True
+    >>> date(3, 5, 6, 8) <= date(4, 5, 6, 8)
+    True
+    >>> date(3, 5, 6, 8) >= date(4, 5, 6, 8)
+    False
+    >>> date(3, 5, 6, 8) > date(4, 5, 6, 8)
+    False
+    """
+    @staticmethod
+    def hms_to_second(hour, minute, second):
+        _SECONDS_PER_HOUR = 3600
+        _SECONDS_PER_MINUTE = 60
+        return (hour * _SECONDS_PER_HOUR + minute * _SECONDS_PER_MINUTE
+                + second)
+
+    @staticmethod
+    def second_to_hms(second):
+        _SECONDS_PER_HOUR = 3600
+        _SECONDS_PER_MINUTE = 60
+        return { 'hour': second // _SECONDS_PER_HOUR,
+                 'minute': (second % _SECONDS_PER_HOUR) // _SECONDS_PER_MINUTE,
+                 'second': second % _SECONDS_PER_MINUTE
+        }
+
+    def __init__(self, year=1, month=1, day=1, hour=0, minute=0, second=0):
+        self._year = year
+        self._month = month
+        self._day = day
+        self._second = self.hms_to_second(hour, minute, second)
+
+    def __str__(self):
+        """
+        >>> str(date(4, 5, 7, second=64))
+        'date(4, 5, 7, 0, 1, 4)'
+        """
+        fmt_str = "date({year:d}, {month:d}, {day:d}, {hour:d}, {minute:d}, {second:d})"
+        return fmt_str.format(year = self.year(),
+                              month = self.month(),
+                              day = self.day(),
+                              hour = self.hour(),
+                              minute = self.minute(),
+                              second = self.second())
+
+    def year(self):
+        return self._year
+
+    def month(self):
+        return self._month
+
+    def day(self):
+        return self._day
+
+    def hour(self):
+        return self.second_to_hms(self._second)['hour']
+
+    def minute(self):
+        return self.second_to_hms(self._second)['minute']
+
+    def second(self):
+        return self.second_to_hms(self._second)['second']
+
+    def second_of_day(self):
+        return self._second
+
+    def __repr__(self):
+        return str(self)
+
+    def __eq__(self, other):
+        return ((self.year() == other.year()) and (self.month() == other.month())
+                and (self.day() == other.day())
+                and (self.second_of_day() == other.second_of_day()))
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __lt__(self, other):
+        if self.year() < other.year():
+            return True
+        elif self.year() > other.year():
+            return False
+        # self.year == other.year
+        if self.month() < other.month():
+            return True
+        elif self.month() > other.month():
+            return False
+        # self.month = other.month
+        if self.day() < other.day():
+            return True
+        elif self.day() > other.day():
+            return False
+        # self.day = other.day
+        if self.second_of_day() < other.second_of_day():
+            return True
+        else:
+            # the dates are equal
+            return False
+
+    def __le__(self, other):
+        return ((self < other) or (self == other))
+
+    def __ge__(self, other):
+        return not (self < other)
+
+    def __gt__(self, other):
+        return not (self <= other)


Test patch: