Repository: ledger/ledger (language: cpp)

Issue:
Bug in handling utf-8
In latest commits from July 20:

Local config : MacOS 12.6 Monterey, Terminal in UTF8.
I guess the fix above is not enough because the July 19 commit works well with the Euro symbol as a commodity :
Python 3.7.14 (default, Sep 10 2022, 11:16:56)
[Clang 13.0.0 (clang-1300.0.29.30)] on darwin
Type "help", "copyright", "credits" or "license" for more information.

```
import ledger
a=ledger.Amount('€1.3')
a
€1.3
```
But with this commit, it's all buggy :

```
ledger.Amount('€1.3')
ﾬ 1
```

See how my Euro symbol has become a Korean syllable!


Discussion / hints:

Fix patch:
diff --git a/python/demo.py b/python/demo.py
index e57af05ae..6a68b1774 100755
--- a/python/demo.py
+++ b/python/demo.py
@@ -55,9 +55,12 @@ def assertEqual(pat, candidate):
 comms = ledger.commodities
 
 usd = comms.find_or_create('$')
-eur = comms.find_or_create('EUR')
 xcd = comms.find_or_create('XCD')
 
+# Tests currency symbols encoded using UCS. For details see #2132.
+eur = comms.find_or_create('€') # UCS-2 / UCS-4
+xxx = comms.find_or_create('¤') # UCS-1
+
 assert not comms.find('CAD')
 assert not comms.has_key('CAD')
 assert not 'CAD' in comms
@@ -79,7 +82,7 @@ def assertEqual(pat, candidate):
 # don't need to worry about them, but they'll show up if you examine all the
 # keys in the commodities dict.
 
-assertEqual([u'', u'$', u'%', u'EUR', u'XCD', u'h', u'm', u's'],
+assertEqual([u'', u'$', u'%', u'XCD', u'h', u'm', u's', u'¤', u'€'],
             sorted(comms.keys()))
 
 # All the styles of dict iteration are supported:
@@ -98,7 +101,7 @@ def assertEqual(pat, candidate):
 # that date.  You can record specific conversion rates for any date using the
 # `exchange' method.
 
-comms.exchange(eur, ledger.Amount('$0.77')) # Trade 1 EUR for $0.77
+comms.exchange(eur, ledger.Amount('$0.77')) # Trade 1 € for $0.77
 comms.exchange(eur, ledger.Amount('$0.66'), datetime.now())
 
 # For the most part, however, you won't be interacting with commodities
@@ -133,7 +136,7 @@ def assertEqual(pat, candidate):
 
 # For addition and subtraction, only amounts of the same commodity may be
 # used, unless one of the amounts has no commodity at all -- in which case the
-# result uses the commodity of the other value.  Adding $10 to 10 EUR, for
+# result uses the commodity of the other value.  Adding $10 to 10 €, for
 # example, causes an ArithmeticError exception, but adding 10 to $10 gives
 # $20.
 
@@ -143,7 +146,7 @@ def assertEqual(pat, candidate):
 assertEqual(zero, one - one)
 
 try:
-    two += ledger.Amount("20 EUR")
+    two += ledger.Amount("20 €")
     assert False
 except ArithmeticError:
     pass
@@ -193,7 +196,7 @@ def assertEqual(pat, candidate):
 # There are several other supported math operations:
 
 amt    = ledger.Amount('$100.12')
-market = ((ledger.Amount('1 EUR') / ledger.Amount('$0.77')) * amt)
+market = ((ledger.Amount('1 €') / ledger.Amount('$0.77')) * amt)
 
 assertEqual(market, amt.value(eur))            # find present market value
 
@@ -222,19 +225,19 @@ def assertEqual(pat, candidate):
 
 # Finally, amounts can be annotated to provide additional information about
 # "lots" of a given commodity.  This example shows $100.12 that was purchased
-# on 2009/10/01 for 140 EUR.  Lot information can be accessed through via the
+# on 2009/10/01 for 140 €.  Lot information can be accessed through via the
 # Amount's `annotation' property.  You can also strip away lot details to get
 # the underlying amount.  If you want the total price of any Amount, by
 # multiplying by its per-unit lot price, call the `Amount.price' method
 # instead of the `Annotation.price' property.
 
-amt2 = ledger.Amount('$100.12 {140 EUR} [2009/10/01]')
+amt2 = ledger.Amount('$100.12 {140 €} [2009/10/01]')
 
 assert amt2.has_annotation()
 assertEqual(amt, amt2.strip_annotations())
 
-assertEqual(ledger.Amount('140 EUR'), amt2.annotation.price)
-assertEqual(ledger.Amount('14016,8 EUR'), amt2.price()) # european amount!
+assertEqual(ledger.Amount('140 €'), amt2.annotation.price)
+assertEqual(ledger.Amount('14016,8 €'), amt2.price()) # european amount!
 
 ###############################################################################
 #
diff --git a/src/py_utils.cc b/src/py_utils.cc
index cf416e08f..8765bb788 100644
--- a/src/py_utils.cc
+++ b/src/py_utils.cc
@@ -134,34 +134,36 @@ struct string_from_python
       PyUnicode_GET_SIZE(obj_ptr);
 #endif
 #if PY_MINOR_VERSION < 12
-    PyUnicode_READY(obj_ptr);
+    if (PyUnicode_READY(obj_ptr))
+      return;
 #endif
-    const char* value;
     switch (PyUnicode_KIND(obj_ptr)) {
-      case PyUnicode_1BYTE_KIND:
-        value = (const char*)PyUnicode_1BYTE_DATA(obj_ptr);
-        str = std::string(value);
-        break;
+      case PyUnicode_1BYTE_KIND: {
+        Py_UCS1* value = PyUnicode_1BYTE_DATA(obj_ptr);
+        if (value == 0) throw_error_already_set();
+        utf8::unchecked::utf16to8(value, value + size, std::back_inserter(str));
+        } break;
 #if PY_MINOR_VERSION < 12 && Py_UNICODE_SIZE == 2
       case PyUnicode_WCHAR_KIND:
 #endif
-      case PyUnicode_2BYTE_KIND:
-        value = (const char*)PyUnicode_2BYTE_DATA(obj_ptr);
+      case PyUnicode_2BYTE_KIND: {
+        Py_UCS2* value = PyUnicode_2BYTE_DATA(obj_ptr);
+        if (value == 0) throw_error_already_set();
         utf8::unchecked::utf16to8(value, value + size, std::back_inserter(str));
-        break;
+        } break;
 #if PY_MINOR_VERSION < 12 && Py_UNICODE_SIZE == 4
       case PyUnicode_WCHAR_KIND:
 #endif
-      case PyUnicode_4BYTE_KIND:
-        value = (const char*)PyUnicode_4BYTE_DATA(obj_ptr);
+      case PyUnicode_4BYTE_KIND: {
+        Py_UCS4* value = PyUnicode_4BYTE_DATA(obj_ptr);
+        if (value == 0) throw_error_already_set();
         utf8::unchecked::utf32to8(value, value + size, std::back_inserter(str));
-        break;
+        } break;
       default:
         assert("PyUnicode_KIND returned an unexpected kind" == NULL);
     }
 #endif // PY_MAJOR_VERSION
 
-    if (value == 0) throw_error_already_set();
     void* storage =
       reinterpret_cast<converter::rvalue_from_python_storage<string> *>
                       (data)->storage.bytes;


Test patch: