Repository: python-babel/babel (python)

Issue:
Weeknumber wrong for 2018-12-31
As mentioned in the docs babel.dates.format_date uses the ISO-Calendar when passing "YYYY" as format string.

Evaluating gives
```
babel.dates.format_date(datetime.datetime(2018,12,31
                                         ), format="w YYYY", locale='en_US')
>> u'53 2018'
```
The correct result should be 1 2019 by iso definitions.

Thanks for your reply

Pull request description:
fixes #619 wrong weeknumber for 31.12.2018
The weeknumber was calculated to 53, but by definition the value
must compute to 1.

the fix will compute the weeknumber by using date.isocalendar if
locale.first_week_day == 0.

Also the computation of the year format 'YYYY' is replaced by isocalendar.

Tests added.

Fixes https://github.com/python-babel/babel/issues/619

Patch:
diff --git a/CHANGES b/CHANGES
index 6d9b0e5..622b4c5 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,19 @@
 Babel Changelog
 ===============
 
+UNRELEASED
+----------
+
+Possibly incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+These may be backward incompatible in some cases, as some more-or-less internal
+APIs have changed. Please feel free to file issues if you bump into anything
+strange and we'll try to help!
+
+* General: Internal uses of ``babel.util.odict`` have been replaced with
+  ``collections.OrderedDict`` from The Python standard library.
+
 Version 2.6.0
 -------------
 
diff --git a/babel/_compat.py b/babel/_compat.py
index 849f749..11b4d7a 100644
--- a/babel/_compat.py
+++ b/babel/_compat.py
@@ -8,7 +8,6 @@ _identity = lambda x: x
 
 if not PY2:
     text_type = str
-    binary_type = bytes
     string_types = (str,)
     integer_types = (int, )
 
@@ -33,7 +32,6 @@ if not PY2:
 
 else:
     text_type = unicode
-    binary_type = str
     string_types = (str, unicode)
     integer_types = (int, long)
 
@@ -63,7 +61,7 @@ number_types = integer_types + (float,)
 def force_text(s, encoding='utf-8', errors='strict'):
     if isinstance(s, text_type):
         return s
-    if isinstance(s, binary_type):
+    if isinstance(s, bytes):
         return s.decode(encoding, errors)
     return text_type(s)
 
diff --git a/babel/dates.py b/babel/dates.py
index 4bae3ea..ec86991 100644
--- a/babel/dates.py
+++ b/babel/dates.py
@@ -1319,9 +1319,7 @@ class DateTimeFormat(object):
     def format_year(self, char, num):
         value = self.value.year
         if char.isupper():
-            week = self.get_week_number(self.get_day_of_year())
-            if week == 0:
-                value -= 1
+            value = self.value.isocalendar()[0]
         year = self.format(value, num)
         if num == 2:
             year = year[-2:]
@@ -1505,8 +1503,20 @@ class DateTimeFormat(object):
         if first_day < 0:
             first_day += 7
         week_number = (day_of_period + first_day - 1) // 7
+
         if 7 - first_day >= self.locale.min_week_days:
             week_number += 1
+        
+        if self.locale.first_week_day == 0:
+            # Correct the weeknumber in case of iso-calendar usage (first_week_day=0). 
+            # If the weeknumber exceeds the maximum number of weeks for the given year
+            # we must count from zero.For example the above calculation gives week 53 
+            # for 2018-12-31. By iso-calender definition 2018 has a max of 52 
+            # weeks, thus the weeknumber must be 53-52=1.
+            max_weeks = date(year=self.value.year, day=28, month=12).isocalendar()[1]
+            if week_number > max_weeks:
+                week_number -= max_weeks
+
         return week_number
 
 
diff --git a/babel/messages/catalog.py b/babel/messages/catalog.py
index 6d61a38..649d0e2 100644
--- a/babel/messages/catalog.py
+++ b/babel/messages/catalog.py
@@ -13,6 +13,7 @@ import re
 import time
 
 from cgi import parse_header
+from collections import OrderedDict
 from datetime import datetime, time as time_
 from difflib import get_close_matches
 from email import message_from_string
@@ -22,7 +23,7 @@ from babel import __version__ as VERSION
 from babel.core import Locale, UnknownLocaleError
 from babel.dates import format_datetime
 from babel.messages.plurals import get_plural
-from babel.util import odict, distinct, LOCALTZ, FixedOffsetTimezone
+from babel.util import distinct, LOCALTZ, FixedOffsetTimezone
 from babel._compat import string_types, number_types, PY2, cmp, text_type, force_text
 
 __all__ = ['Message', 'Catalog', 'TranslationError']
@@ -269,7 +270,7 @@ class Catalog(object):
         self.domain = domain
         self.locale = locale
         self._header_comment = header_comment
-        self._messages = odict()
+        self._messages = OrderedDict()
 
         self.project = project or 'PROJECT'
         self.version = version or 'VERSION'
@@ -295,7 +296,7 @@ class Catalog(object):
         self.revision_date = revision_date
         self.fuzzy = fuzzy
 
-        self.obsolete = odict()  # Dictionary of obsolete messages
+        self.obsolete = OrderedDict()  # Dictionary of obsolete messages
         self._num_plurals = None
         self._plural_expr = None
 
@@ -754,7 +755,7 @@ class Catalog(object):
         """
         messages = self._messages
         remaining = messages.copy()
-        self._messages = odict()
+        self._messages = OrderedDict()
 
         # Prepare for fuzzy matching
         fuzzy_candidates = []
diff --git a/babel/messages/frontend.py b/babel/messages/frontend.py
index 16cb900..f673f90 100644
--- a/babel/messages/frontend.py
+++ b/babel/messages/frontend.py
@@ -17,6 +17,7 @@ import re
 import shutil
 import sys
 import tempfile
+from collections import OrderedDict
 from datetime import datetime
 from locale import getpreferredencoding
 
@@ -28,7 +29,7 @@ from babel.messages.catalog import Catalog
 from babel.messages.extract import DEFAULT_KEYWORDS, DEFAULT_MAPPING, check_and_call_extract_file, extract_from_dir
 from babel.messages.mofile import write_mo
 from babel.messages.pofile import read_po, write_po
-from babel.util import LOCALTZ, odict
+from babel.util import LOCALTZ
 from distutils import log as distutils_log
 from distutils.cmd import Command as _Command
 from distutils.errors import DistutilsOptionError, DistutilsSetupError
@@ -646,6 +647,8 @@ class update_catalog(Command):
         ('output-file=', 'o',
          "name of the output file (default "
          "'<output_dir>/<locale>/LC_MESSAGES/<domain>.po')"),
+        ('omit-header', None,
+         "do not include msgid "" entry in header"),
         ('locale=', 'l',
          'locale of the catalog to compile'),
         ('width=', 'w',
@@ -660,15 +663,19 @@ class update_catalog(Command):
         ('update-header-comment', None,
          'update target header comment'),
         ('previous', None,
-         'keep previous msgids of translated messages')
+         'keep previous msgids of translated messages'),
+    ]
+    boolean_options = [
+        'omit-header', 'no-wrap', 'ignore-obsolete', 'no-fuzzy-matching',
+        'previous', 'update-header-comment',
     ]
-    boolean_options = ['no-wrap', 'ignore-obsolete', 'no-fuzzy-matching', 'previous', 'update-header-comment']
 
     def initialize_options(self):
         self.domain = 'messages'
         self.input_file = None
         self.output_dir = None
         self.output_file = None
+        self.omit_header = False
         self.locale = None
         self.width = None
         self.no_wrap = False
@@ -739,6 +746,7 @@ class update_catalog(Command):
             try:
                 with open(tmpname, 'wb') as tmpfile:
                     write_po(tmpfile, catalog,
+                             omit_header=self.omit_header,
                              ignore_obsolete=self.ignore_obsolete,
                              include_previous=self.previous, width=self.width)
             except:
@@ -965,7 +973,7 @@ def parse_mapping(fileobj, filename=None):
     options_map = {}
 
     parser = RawConfigParser()
-    parser._sections = odict(parser._sections)  # We need ordered sections
+    parser._sections = OrderedDict(parser._sections)  # We need ordered sections
 
     if PY2:
         parser.readfp(fileobj, filename)
diff --git a/babel/numbers.py b/babel/numbers.py
index 4306e93..e5650dd 100644
--- a/babel/numbers.py
+++ b/babel/numbers.py
@@ -448,7 +448,7 @@ def format_currency(
     >>> format_currency(1099.98, 'JPY', locale='en_US')
     u'\\xa51,100'
     >>> format_currency(1099.98, 'COP', u'#,##0.00', locale='es_ES')
-    u'1.100'
+    u'1.099,98'
 
     However, the number of decimal digits can be overriden from the currency
     information, by setting the last parameter to ``False``:
diff --git a/babel/units.py b/babel/units.py
index be851c2..e58bf81 100644
--- a/babel/units.py
+++ b/babel/units.py
@@ -82,7 +82,7 @@ def format_unit(value, measurement_unit, length='long', format=None, locale=LC_N
 
     >>> from babel._compat import decimal
     >>> format_unit(decimal.Decimal("-42.774"), 'temperature-celsius', 'short', format='#.0', locale='fr')
-    u'-42,8 \\xb0C'
+    u'-42,8\\u202f\\xb0C'
 
     The locale's usual pluralization rules are respected.
 
diff --git a/babel/util.py b/babel/util.py
index fb93f16..c443b69 100644
--- a/babel/util.py
+++ b/babel/util.py
@@ -10,6 +10,7 @@
 """
 
 import codecs
+import collections
 from datetime import timedelta, tzinfo
 import os
 import re
@@ -220,77 +221,8 @@ def wraptext(text, width=70, initial_indent='', subsequent_indent=''):
     return wrapper.wrap(text)
 
 
-class odict(dict):
-    """Ordered dict implementation.
-
-    :see: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/107747
-    """
-
-    def __init__(self, data=None):
-        dict.__init__(self, data or {})
-        self._keys = list(dict.keys(self))
-
-    def __delitem__(self, key):
-        dict.__delitem__(self, key)
-        self._keys.remove(key)
-
-    def __setitem__(self, key, item):
-        new_key = key not in self
-        dict.__setitem__(self, key, item)
-        if new_key:
-            self._keys.append(key)
-
-    def __iter__(self):
-        return iter(self._keys)
-    iterkeys = __iter__
-
-    def clear(self):
-        dict.clear(self)
-        self._keys = []
-
-    def copy(self):
-        d = odict()
-        d.update(self)
-        return d
-
-    def items(self):
-        return zip(self._keys, self.values())
-
-    def iteritems(self):
-        return izip(self._keys, self.itervalues())
-
-    def keys(self):
-        return self._keys[:]
-
-    def pop(self, key, default=missing):
-        try:
-            value = dict.pop(self, key)
-            self._keys.remove(key)
-            return value
-        except KeyError as e:
-            if default == missing:
-                raise e
-            else:
-                return default
-
-    def popitem(self, key):
-        self._keys.remove(key)
-        return dict.popitem(key)
-
-    def setdefault(self, key, failobj=None):
-        dict.setdefault(self, key, failobj)
-        if key not in self._keys:
-            self._keys.append(key)
-
-    def update(self, dict):
-        for (key, val) in dict.items():
-            self[key] = val
-
-    def values(self):
-        return map(self.get, self._keys)
-
-    def itervalues(self):
-        return imap(self.get, self._keys)
+# TODO (Babel 3.x): Remove this re-export
+odict = collections.OrderedDict
 
 
 class FixedOffsetTimezone(tzinfo):
diff --git a/scripts/download_import_cldr.py b/scripts/download_import_cldr.py
index b6e6f43..f0d9b10 100755
--- a/scripts/download_import_cldr.py
+++ b/scripts/download_import_cldr.py
@@ -13,9 +13,9 @@ except ImportError:
     from urllib import urlretrieve
 
 
-URL = 'https://unicode.org/Public/cldr/33/core.zip'
-FILENAME = 'core-33.0.zip'
-FILESUM = 'fa3490082c086d21257153609642f54fcf788fcfda4966fe67f3f6daca0d58b9'
+URL = 'https://unicode.org/Public/cldr/34/core.zip'
+FILENAME = 'cldr-core-34.0.zip'
+FILESUM = '1d79f6ce294ff350a3ee59d7dd69451fd24776f7105fd6a55c255ffb813c03ba'
 BLKSIZE = 131072
 
 


Test patch:
diff --git a/tests/messages/test_pofile.py b/tests/messages/test_pofile.py
index 9872e9d..a6973d7 100644
--- a/tests/messages/test_pofile.py
+++ b/tests/messages/test_pofile.py
@@ -13,7 +13,6 @@
 
 from datetime import datetime
 import unittest
-import sys
 
 from babel.core import Locale
 from babel.messages.catalog import Catalog, Message
diff --git a/tests/test_dates.py b/tests/test_dates.py
index b8c293b..d77c0ea 100644
--- a/tests/test_dates.py
+++ b/tests/test_dates.py
@@ -79,6 +79,15 @@ class DateTimeFormatTestCase(unittest.TestCase):
         fmt = dates.DateTimeFormat(d, locale='en_US')
         self.assertEqual('53', fmt['w'])
 
+    def test_week_of_year_de_first_us_last_with_year(self):
+        d = date(2018,12,31)
+        fmt = dates.DateTimeFormat(d, locale='de_DE')
+        self.assertEqual('1', fmt['w'])
+        self.assertEqual('2019', fmt['YYYY'])
+        fmt = dates.DateTimeFormat(d, locale='en_US')
+        self.assertEqual('53', fmt['w'])
+        self.assertEqual('2018',fmt['yyyy'])
+
     def test_week_of_month_first(self):
         d = date(2006, 1, 8)
         fmt = dates.DateTimeFormat(d, locale='de_DE')
diff --git a/tests/test_localedata.py b/tests/test_localedata.py
index 37c1304..dceb984 100644
--- a/tests/test_localedata.py
+++ b/tests/test_localedata.py
@@ -15,7 +15,7 @@ import unittest
 import random
 from operator import methodcaller
 
-from babel import localedata, numbers
+from babel import localedata
 
 class MergeResolveTestCase(unittest.TestCase):
 
diff --git a/tests/test_numbers.py b/tests/test_numbers.py
index d0f24bd..99aa96b 100644
--- a/tests/test_numbers.py
+++ b/tests/test_numbers.py
@@ -16,11 +16,10 @@ import pytest
 
 from datetime import date
 
-from babel import Locale, localedata, numbers
+from babel import localedata, numbers
 from babel.numbers import (
     list_currencies, validate_currency, UnknownCurrencyError, is_currency, normalize_currency,
     get_currency_precision, get_decimal_precision, get_currency_unit_pattern)
-from babel.localedata import locale_identifiers
 from babel._compat import decimal
 
 
@@ -209,7 +208,7 @@ def test_list_currencies():
 
     assert list_currencies(locale='pa_Arab') == {'PKR', 'INR', 'EUR'}
 
-    assert len(list_currencies()) == 300
+    assert len(list_currencies()) == 303
 
 
 def test_validate_currency():
@@ -412,7 +411,7 @@ def test_format_currency_format_type():
     assert (numbers.format_currency(1099.98, 'JPY', locale='en_US')
             == u'\xa51,100')
     assert (numbers.format_currency(1099.98, 'COP', u'#,##0.00', locale='es_ES')
-            == u'1.100')
+            == u'1.099,98')
     assert (numbers.format_currency(1099.98, 'JPY', locale='en_US',
                                     currency_digits=False)
             == u'\xa51,099.98')
diff --git a/tests/test_util.py b/tests/test_util.py
index ef59110..0fc59e5 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -38,20 +38,6 @@ def test_pathmatch():
     assert not util.pathmatch('./foo/**.py', 'blah/foo/bar/baz.py')
 
 
-def test_odict_pop():
-    odict = util.odict()
-    odict[0] = 1
-    value = odict.pop(0)
-    assert 1 == value
-    assert [] == list(odict.items())
-    assert odict.pop(2, None) is None
-    try:
-        odict.pop(2)
-        assert False
-    except KeyError:
-        assert True
-
-
 class FixedOffsetTimezoneTestCase(unittest.TestCase):
 
     def test_zone_negative_offset(self):