Repository: dateutil/dateutil (Python)

Issue:
tzstr not properly detecting invalid strings
I noticed that [this test](https://github.com/dateutil/dateutil/blob/master/dateutil/test/test_tz.py#L903) is not actually parsing the time zone string correctly because that string format is not supported:

``` python
>>> from dateutil import tz
>>> tz.tzstr("EST5EDT") == tz.tzstr("EST5EDT,4,1,0,7200,10,-1,0,7200,3600")
True
>>> tz.tzstr("EST5EDT") == tz.tzstr("EST5EDT,5,4,0,7200,11,-3,0,7200,3600")
True
```

I'm thinking maybe once this is fixed, these tests should be switched to compare tzstr to tzrange, or to some performance-based tests (like checking that the transitions happen at the right time, etc.



Hints:
Putting this off for the first bugfix release.


Patch:
diff --git a/dateutil/parser/_parser.py b/dateutil/parser/_parser.py
--- a/dateutil/parser/_parser.py
+++ b/dateutil/parser/_parser.py
@@ -44,6 +44,8 @@
 
 from decimal import Decimal
 
+import re
+
 from .. import relativedelta
 from .. import tz
 
@@ -620,7 +622,7 @@ def parse(self, timestr, default=None,
     class _result(_resultbase):
         __slots__ = ["year", "month", "day", "weekday",
                      "hour", "minute", "second", "microsecond",
-                     "tzname", "tzoffset", "ampm"]
+                     "tzname", "tzoffset", "ampm","any_unused_tokens"]
 
     def _parse(self, timestr, dayfirst=None, yearfirst=None, fuzzy=False,
                fuzzy_with_tokens=False):
@@ -1311,7 +1313,8 @@ def __init__(self):
 
     def parse(self, tzstr):
         res = self._result()
-        l = _timelex.split(tzstr)
+        l = [x for x in re.split(r'([,:.]|[a-zA-Z]+|[0-9]+)',tzstr) if x]
+        used_idxs = list()
         try:
 
             len_l = len(l)
@@ -1330,6 +1333,9 @@ def parse(self, tzstr):
                     else:
                         offattr = "dstoffset"
                         res.dstabbr = "".join(l[i:j])
+
+                    for ii in range(j):
+                        used_idxs.append(ii)
                     i = j
                     if (i < len_l and (l[i] in ('+', '-') or l[i][0] in
                                        "0123456789")):
@@ -1337,6 +1343,7 @@ def parse(self, tzstr):
                             # Yes, that's right.  See the TZ variable
                             # documentation.
                             signal = (1, -1)[l[i] == '+']
+                            used_idxs.append(i)
                             i += 1
                         else:
                             signal = -1
@@ -1350,6 +1357,7 @@ def parse(self, tzstr):
                             setattr(res, offattr,
                                     (int(l[i]) * 3600 +
                                      int(l[i + 2]) * 60) * signal)
+                            used_idxs.append(i)
                             i += 2
                         elif len_li <= 2:
                             # -[0]3
@@ -1357,12 +1365,14 @@ def parse(self, tzstr):
                                     int(l[i][:2]) * 3600 * signal)
                         else:
                             return None
+                        used_idxs.append(i)
                         i += 1
                     if res.dstabbr:
                         break
                 else:
                     break
 
+
             if i < len_l:
                 for j in range(i, len_l):
                     if l[j] == ';':
@@ -1376,32 +1386,39 @@ def parse(self, tzstr):
                 pass
             elif (8 <= l.count(',') <= 9 and
                   not [y for x in l[i:] if x != ','
-                       for y in x if y not in "0123456789"]):
+                       for y in x if y not in "0123456789+-"]):
                 # GMT0BST,3,0,30,3600,10,0,26,7200[,3600]
                 for x in (res.start, res.end):
                     x.month = int(l[i])
+                    used_idxs.append(i)
                     i += 2
                     if l[i] == '-':
                         value = int(l[i + 1]) * -1
+                        used_idxs.append(i)
                         i += 1
                     else:
                         value = int(l[i])
+                    used_idxs.append(i)
                     i += 2
                     if value:
                         x.week = value
                         x.weekday = (int(l[i]) - 1) % 7
                     else:
                         x.day = int(l[i])
+                    used_idxs.append(i)
                     i += 2
                     x.time = int(l[i])
+                    used_idxs.append(i)
                     i += 2
                 if i < len_l:
                     if l[i] in ('-', '+'):
                         signal = (-1, 1)[l[i] == "+"]
+                        used_idxs.append(i)
                         i += 1
                     else:
                         signal = 1
-                    res.dstoffset = (res.stdoffset + int(l[i])) * signal
+                    used_idxs.append(i)
+                    res.dstoffset = (res.stdoffset + int(l[i]) * signal)
             elif (l.count(',') == 2 and l[i:].count('/') <= 2 and
                   not [y for x in l[i:] if x not in (',', '/', 'J', 'M',
                                                      '.', '-', ':')
@@ -1409,29 +1426,37 @@ def parse(self, tzstr):
                 for x in (res.start, res.end):
                     if l[i] == 'J':
                         # non-leap year day (1 based)
+                        used_idxs.append(i)
                         i += 1
                         x.jyday = int(l[i])
                     elif l[i] == 'M':
                         # month[-.]week[-.]weekday
+                        used_idxs.append(i)
                         i += 1
                         x.month = int(l[i])
+                        used_idxs.append(i)
                         i += 1
                         assert l[i] in ('-', '.')
+                        used_idxs.append(i)
                         i += 1
                         x.week = int(l[i])
                         if x.week == 5:
                             x.week = -1
+                        used_idxs.append(i)
                         i += 1
                         assert l[i] in ('-', '.')
+                        used_idxs.append(i)
                         i += 1
                         x.weekday = (int(l[i]) - 1) % 7
                     else:
                         # year day (zero based)
                         x.yday = int(l[i]) + 1
 
+                    used_idxs.append(i)
                     i += 1
 
                     if i < len_l and l[i] == '/':
+                        used_idxs.append(i)
                         i += 1
                         # start time
                         len_li = len(l[i])
@@ -1442,8 +1467,10 @@ def parse(self, tzstr):
                         elif i + 1 < len_l and l[i + 1] == ':':
                             # -03:00
                             x.time = int(l[i]) * 3600 + int(l[i + 2]) * 60
+                            used_idxs.append(i)
                             i += 2
                             if i + 1 < len_l and l[i + 1] == ':':
+                                used_idxs.append(i)
                                 i += 2
                                 x.time += int(l[i])
                         elif len_li <= 2:
@@ -1451,6 +1478,7 @@ def parse(self, tzstr):
                             x.time = (int(l[i][:2]) * 3600)
                         else:
                             return None
+                        used_idxs.append(i)
                         i += 1
 
                     assert i == len_l or l[i] == ','
@@ -1462,6 +1490,8 @@ def parse(self, tzstr):
         except (IndexError, ValueError, AssertionError):
             return None
 
+        unused_idxs = set(range(len_l)).difference(used_idxs)
+        res.any_unused_tokens = not {l[n] for n in unused_idxs}.issubset({",",":"})
         return res
 
 
diff --git a/dateutil/tz/tz.py b/dateutil/tz/tz.py
--- a/dateutil/tz/tz.py
+++ b/dateutil/tz/tz.py
@@ -993,7 +993,7 @@ def __init__(self, s, posix_offset=False):
         self._s = s
 
         res = parser._parsetz(s)
-        if res is None:
+        if res is None or res.any_unused_tokens:
             raise ValueError("unknown string format")
 
         # Here we break the compatibility with the TZ variable handling.


Test patch:
diff --git a/dateutil/test/test_tz.py b/dateutil/test/test_tz.py
--- a/dateutil/test/test_tz.py
+++ b/dateutil/test/test_tz.py
@@ -22,7 +22,7 @@
 import pytest
 
 # dateutil imports
-from dateutil.relativedelta import relativedelta, SU
+from dateutil.relativedelta import relativedelta, SU, TH
 from dateutil.parser import parse
 from dateutil import tz as tz
 from dateutil import zoneinfo
@@ -1126,6 +1126,7 @@ def testRangeInequalityUnsupported(self):
         self.assertFalse(TZR != ComparesEqual)
 
 
+@pytest.mark.tzstr
 class TZStrTest(unittest.TestCase, TzFoldMixin):
     # POSIX string indicating change to summer time on the 2nd Sunday in March
     # at 2AM, and ending the 1st Sunday in November at 2AM. (valid >= 2007)
@@ -1146,6 +1147,14 @@ def gettz(self, tzname):
 
         return tz.tzstr(tzname_map[tzname])
 
+
+    def test_internal_timedeltas(self):
+        self.assertNotEqual(tz.tzstr("EST5EDT,5,4,0,7200,11,-3,0,7200")._start_delta,
+                            tz.tzstr("EST5EDT,4,1,0,7200,10,-1,0,7200")._start_delta)
+
+        self.assertNotEqual(tz.tzstr("EST5EDT,5,4,0,7200,11,-3,0,7200")._end_delta,
+                            tz.tzstr("EST5EDT,4,1,0,7200,10,-1,0,7200")._end_delta)
+
     def testStrStart1(self):
         self.assertEqual(datetime(2003, 4, 6, 1, 59,
                                   tzinfo=tz.tzstr("EST5EDT")).tzname(), "EST")
@@ -1157,7 +1166,7 @@ def testStrEnd1(self):
                                   tzinfo=tz.tzstr("EST5EDT")).tzname(), "EDT")
 
         end = tz.enfold(datetime(2003, 10, 26, 1, 00,
-                                  tzinfo=tz.tzstr("EST5EDT")), fold=1)
+                                 tzinfo=tz.tzstr("EST5EDT")), fold=1)
         self.assertEqual(end.tzname(), "EST")
 
     def testStrStart2(self):
@@ -1173,7 +1182,7 @@ def testStrEnd2(self):
                                   tzinfo=tz.tzstr(s)).tzname(), "EDT")
 
         end = tz.enfold(datetime(2003, 10, 26, 1, 00,
-                                  tzinfo=tz.tzstr(s)), fold=1)
+                                 tzinfo=tz.tzstr(s)), fold=1)
         self.assertEqual(end.tzname(), "EST")
 
     def testStrStart3(self):
@@ -1189,7 +1198,7 @@ def testStrEnd3(self):
                                   tzinfo=tz.tzstr(s)).tzname(), "EDT")
 
         end = tz.enfold(datetime(2003, 10, 26, 1, 00,
-                                  tzinfo=tz.tzstr(s)), fold=1)
+                                 tzinfo=tz.tzstr(s)), fold=1)
         self.assertEqual(end.tzname(), "EST")
 
     def testStrStart4(self):
@@ -1219,7 +1228,7 @@ def testStrEnd5(self):
         self.assertEqual(datetime(2003, 10, 26, 0, 59,
                                   tzinfo=tz.tzstr(s)).tzname(), "EDT")
         end = tz.enfold(datetime(2003, 10, 26, 1, 00,
-                                  tzinfo=tz.tzstr(s)), fold=1)
+                                 tzinfo=tz.tzstr(s)), fold=1)
         self.assertEqual(end.tzname(), "EST")
 
     def testStrStart6(self):
@@ -1313,6 +1322,113 @@ def testTzStrFailure(self):
             tz.tzstr('InvalidString;439999')
 
 
+@pytest.mark.tzstr
+@pytest.mark.parametrize('tz_str,expected', [
+    # From https://www.gnu.org/software/libc/manual/html_node/TZ-Variable.html
+    ('', tz.tzrange(None)),     # TODO: Should change this so tz.tzrange('') works
+    ('EST+5EDT,M3.2.0/2,M11.1.0/12',
+     tz.tzrange('EST', -18000, 'EDT', -14400,
+        start=relativedelta(month=3, day=1, weekday=SU(2), hours=2),
+        end=relativedelta(month=11, day=1, weekday=SU(1), hours=11))),
+    ('WART4WARST,J1/0,J365/25',  # This is DST all year, Western Argentina Summer Time
+     tz.tzrange('WART', timedelta(hours=-4), 'WARST',
+        start=relativedelta(month=1, day=1, hours=0),
+        end=relativedelta(month=12, day=31, days=1))),
+    ('IST-2IDT,M3.4.4/26,M10.5.0',      # Israel Standard / Daylight Time
+     tz.tzrange('IST', timedelta(hours=2), 'IDT',
+        start=relativedelta(month=3, day=1, weekday=TH(4), days=1, hours=2),
+        end=relativedelta(month=10, day=31, weekday=SU(-1), hours=1))),
+    ('WGT3WGST,M3.5.0/2,M10.5.0/1',
+     tz.tzrange('WGT', timedelta(hours=-3), 'WGST',
+        start=relativedelta(month=3, day=31, weekday=SU(-1), hours=2),
+        end=relativedelta(month=10, day=31, weekday=SU(-1), hours=0))),
+
+    # Different offset specifications
+    ('WGT0300WGST',
+     tz.tzrange('WGT', timedelta(hours=-3), 'WGST')),
+    ('WGT03:00WGST',
+     tz.tzrange('WGT', timedelta(hours=-3), 'WGST')),
+    ('AEST-1100AEDT',
+     tz.tzrange('AEST', timedelta(hours=11), 'AEDT')),
+    ('AEST-11:00AEDT',
+     tz.tzrange('AEST', timedelta(hours=11), 'AEDT')),
+
+    # Different time formats
+    ('EST5EDT,M3.2.0/4:00,M11.1.0/3:00',
+     tz.tzrange('EST', timedelta(hours=-5), 'EDT',
+        start=relativedelta(month=3, day=1, weekday=SU(2), hours=4),
+        end=relativedelta(month=11, day=1, weekday=SU(1), hours=2))),
+    ('EST5EDT,M3.2.0/04:00,M11.1.0/03:00',
+     tz.tzrange('EST', timedelta(hours=-5), 'EDT',
+        start=relativedelta(month=3, day=1, weekday=SU(2), hours=4),
+        end=relativedelta(month=11, day=1, weekday=SU(1), hours=2))),
+    ('EST5EDT,M3.2.0/0400,M11.1.0/0300',
+     tz.tzrange('EST', timedelta(hours=-5), 'EDT',
+        start=relativedelta(month=3, day=1, weekday=SU(2), hours=4),
+        end=relativedelta(month=11, day=1, weekday=SU(1), hours=2))),
+])
+def test_valid_GNU_tzstr(tz_str, expected):
+    tzi = tz.tzstr(tz_str)
+
+    assert tzi == expected
+
+
+@pytest.mark.tzstr
+@pytest.mark.parametrize('tz_str, expected', [
+    ('EST5EDT,5,4,0,7200,11,3,0,7200',
+     tz.tzrange('EST', timedelta(hours=-5), 'EDT',
+        start=relativedelta(month=5, day=1, weekday=SU(+4), hours=+2),
+        end=relativedelta(month=11, day=1, weekday=SU(+3), hours=+1))),
+    ('EST5EDT,5,-4,0,7200,11,3,0,7200',
+     tz.tzrange('EST', timedelta(hours=-5), 'EDT',
+        start=relativedelta(hours=+2, month=5, day=31, weekday=SU(-4)),
+        end=relativedelta(hours=+1, month=11, day=1, weekday=SU(+3)))),
+    ('EST5EDT,5,4,0,7200,11,-3,0,7200',
+     tz.tzrange('EST', timedelta(hours=-5), 'EDT',
+        start=relativedelta(hours=+2, month=5, day=1, weekday=SU(+4)),
+        end=relativedelta(hours=+1, month=11, day=31, weekday=SU(-3)))),
+    ('EST5EDT,5,4,0,7200,11,-3,0,7200,3600',
+     tz.tzrange('EST', timedelta(hours=-5), 'EDT',
+        start=relativedelta(hours=+2, month=5, day=1, weekday=SU(+4)),
+        end=relativedelta(hours=+1, month=11, day=31, weekday=SU(-3)))),
+    ('EST5EDT,5,4,0,7200,11,-3,0,7200,3600',
+     tz.tzrange('EST', timedelta(hours=-5), 'EDT',
+        start=relativedelta(hours=+2, month=5, day=1, weekday=SU(+4)),
+        end=relativedelta(hours=+1, month=11, day=31, weekday=SU(-3)))),
+    ('EST5EDT,5,4,0,7200,11,-3,0,7200,-3600',
+     tz.tzrange('EST', timedelta(hours=-5), 'EDT', timedelta(hours=-6),
+        start=relativedelta(hours=+2, month=5, day=1, weekday=SU(+4)),
+        end=relativedelta(hours=+3, month=11, day=31, weekday=SU(-3)))),
+    ('EST5EDT,5,4,0,7200,11,-3,0,7200,+7200',
+     tz.tzrange('EST', timedelta(hours=-5), 'EDT', timedelta(hours=-3),
+        start=relativedelta(hours=+2, month=5, day=1, weekday=SU(+4)),
+        end=relativedelta(hours=0, month=11, day=31, weekday=SU(-3)))),
+    ('EST5EDT,5,4,0,7200,11,-3,0,7200,+3600',
+     tz.tzrange('EST', timedelta(hours=-5), 'EDT',
+        start=relativedelta(hours=+2, month=5, day=1, weekday=SU(+4)),
+        end=relativedelta(hours=+1, month=11, day=31, weekday=SU(-3)))),
+])
+def test_valid_default_format(tz_str, expected):
+    # This tests the "default" format that is used widely in the tests and
+    # examples. It is unclear where this format originated from.
+    tzi = tz.tzstr(tz_str)
+    assert tzi == expected
+
+
+@pytest.mark.tzstr
+@pytest.mark.parametrize('tz_str', [
+    'hdfiughdfuig,dfughdfuigpu87ñ::',
+    ',dfughdfuigpu87ñ::',
+    '-1:WART4WARST,J1,J365/25',
+    'WART4WARST,J1,J365/-25',
+    'IST-2IDT,M3.4.-1/26,M10.5.0',
+    'IST-2IDT,M3,2000,1/26,M10,5,0'
+])
+def test_invalid_GNU_tzstr(tz_str):
+    with pytest.raises(ValueError):
+        tz.tzstr(tz_str)
+
+
 class TZICalTest(unittest.TestCase, TzFoldMixin):
     def _gettz_str_tuple(self, tzname):
         TZ_EST = (