Repository: randombit/botan
Pull request #3121 (2022-12-15 12:04:37)

Fix incorrect output of negative BigInts (GH #3073)


Patch:
diff --git a/src/lib/math/bigint/big_io.cpp b/src/lib/math/bigint/big_io.cpp
index ae2855fcc..7314e5c02 100644
--- a/src/lib/math/bigint/big_io.cpp
+++ b/src/lib/math/bigint/big_io.cpp
@@ -22,22 +22,15 @@ std::ostream& operator<<(std::ostream& stream, const BigInt& n)
 
    const size_t base = (stream_flags & std::ios::hex) ? 16 : 10;
 
-   if(n.is_zero())
-      stream.write("0", 1);
-   else
-      {
-      if(n.is_negative())
-         stream.write("-", 1);
+   std::string enc;
 
-      std::string enc;
+   if(base == 10)
+      enc = n.to_dec_string();
+   else
+      enc = n.to_hex_string();
 
-      if(base == 10)
-         enc = n.to_dec_string();
-      else
-         enc = n.to_hex_string();
+   stream.write(enc.data(), enc.size());
 
-      stream.write(enc.data(), enc.size());
-      }
    if(!stream.good())
       throw Stream_IO_Error("BigInt output operator has failed");
    return stream;
diff --git a/src/lib/math/bigint/bigint.h b/src/lib/math/bigint/bigint.h
index cd7195e73..84ca2b46e 100644
--- a/src/lib/math/bigint/bigint.h
+++ b/src/lib/math/bigint/bigint.h
@@ -500,11 +500,22 @@ class BOTAN_PUBLIC_API(2,0) BigInt final
      /**
      * Convert this value to a decimal string.
      * Warning: decimal conversions are relatively slow
+     *
+     * If the integer is zero then "0" is returned.
+     * If the integer is negative then "-" is prefixed.
      */
      std::string to_dec_string() const;
 
      /**
      * Convert this value to a hexadecimal string.
+     *
+     * If the integer is negative then "-" is prefixed.
+     * Then a prefix of "0x" is added.
+     * Follows is a sequence of hexadecimal characters in uppercase.
+     *
+     * The number of hexadecimal characters is always an even number,
+     * with a zero prefix being included if necessary.
+     * For example encoding the integer "5" results in "0x05"
      */
      std::string to_hex_string() const;
 


Test patch:
diff --git a/src/tests/test_bigint.cpp b/src/tests/test_bigint.cpp
index 4e76524d0..05945745a 100644
--- a/src/tests/test_bigint.cpp
+++ b/src/tests/test_bigint.cpp
@@ -200,17 +200,41 @@ class BigInt_Unit_Tests final : public Test
             result.test_eq("input '" + vec.first + "'", n, vec.second);
             }
 
-         BigInt n = 33;
+         auto check_bigint_formatting = [&](const Botan::BigInt& n,
+                                            const std::string& dec,
+                                            const std::string& hex,
+                                            const std::string& neg_dec,
+                                            const std::string& neg_hex)
+            {
+            std::ostringstream oss;
+            oss << n;
+            result.test_eq("output decimal", oss.str(), dec);
+
+            oss.str("");
+            oss << (-n);
+            result.test_eq("output negative decimal", oss.str(), neg_dec);
 
-         std::ostringstream oss;
-         oss << n;
-         result.test_eq("output 33 dec", oss.str(), "33");
+            oss.str("");
+            oss << std::hex << n;
+            result.test_eq("output hex", oss.str(), hex);
 
-         oss.str("");
-         oss << std::hex << n;
-         result.test_eq("output 33 hex", oss.str(), "0x21");
+            oss.str("");
+            oss << std::hex << (-n);
+            result.test_eq("output negative hex", oss.str(), neg_hex);
+            };
 
-         result.test_throws("octal output not supported", [&]() { oss << std::oct << n; });
+         check_bigint_formatting(Botan::BigInt(33), "33", "0x21", "-33", "-0x21");
+         check_bigint_formatting(Botan::BigInt::from_s32(-33), "-33", "-0x21", "33", "0x21");
+         check_bigint_formatting(Botan::BigInt(255), "255", "0xFF", "-255", "-0xFF");
+         check_bigint_formatting(Botan::BigInt(0), "0", "0x00", "0", "0x00");
+         check_bigint_formatting(Botan::BigInt(5), "5", "0x05", "-5", "-0x05");
+
+         result.test_throws("octal output not supported", [&]()
+            {
+            Botan::BigInt n(5);
+            std::ostringstream oss;
+            oss << std::oct << n;
+            });
 
          return result;
          }