Repository: ergoplatform/sigma-rust (language: rust)

Issue:
Preserve ContextExtension items order to\from JSON encoding
Discovered in #677


Discussion / hints:

Fix patch:
diff --git a/Cargo.toml b/Cargo.toml
index 0dc3a4486..7bfea3e90 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -52,9 +52,9 @@ lazy_static = "1.4"
 bs58 = "0.4.0"
 base16 = "0.2.1"
 base64 = "0.13.0"
-indexmap = "1.3.2"
+indexmap = {version ="1.3.2", features = ["serde"]}
 serde = { version = "1.0", features = ["derive"] }
-serde_json = { version = "1.0", features = ["arbitrary_precision"] }
+serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] }
 serde_with = { version = "1.9.1", features = ["json"] }
 rand = "0.8.5"
 bytes = "1.1"
diff --git a/ergo-lib/src/chain/json/context_extension.rs b/ergo-lib/src/chain/json/context_extension.rs
index fbb14a226..f75411efc 100644
--- a/ergo-lib/src/chain/json/context_extension.rs
+++ b/ergo-lib/src/chain/json/context_extension.rs
@@ -2,12 +2,11 @@ use ergotree_interpreter::sigma_protocol::prover::ContextExtension;
 use ergotree_ir::{mir::constant::Constant, serialization::SigmaSerializable};
 use indexmap::IndexMap;
 use serde::{ser::SerializeMap, Deserialize, Serialize};
-use std::collections::HashMap;
 
 #[cfg_attr(
     feature = "json",
     derive(Deserialize),
-    serde(try_from = "HashMap<String, String>"),
+    serde(try_from = "indexmap::IndexMap<String, String>"),
     serde(remote = "ContextExtension")
 )]
 #[derive(Debug, PartialEq, Eq, Clone)]
@@ -42,6 +41,10 @@ impl From<ContextExtension> for ContextExtensionSerde {
 #[cfg(test)]
 #[allow(clippy::unwrap_used)]
 mod tests {
+    use ergo_chain_types::Digest32;
+
+    use crate::chain::transaction::Transaction;
+
     use super::*;
 
     #[test]
@@ -62,4 +65,87 @@ mod tests {
         assert!(c.values.get(&1u8).is_some());
         assert!(c.values.get(&3u8).is_some());
     }
+
+    #[test]
+    fn item_order_preservation_685() {
+        let tx_json = r#"
+{
+        "id" : "c8520befd345ff40fcf244b44ffe8cea29c8b116b174cfaf4f2a521604d531a4",
+        "inputs" : [
+          {
+            "boxId" : "59f2856068c56264d290520043044ace138a3a80d414748d0e4dcd0806188546",
+            "spendingProof" : {
+              "proofBytes" : "",
+              "extension" : {
+                "0" : "04c60f",
+                "5" : "0514",
+                "10" : "0eee03101808cd0279aed8dea2b2a25316d5d49d13bf51c0b2c1dc696974bb4b0c07b5894e998e56040005e0e0a447040404060402040004000e201d5afc59838920bb5ef2a8f9d63825a55b1d48e269d7cecee335d637c3ff5f3f0e20003bd19d0187117f130b62e1bcab0939929ff5c7709f843c5c4dd158949285d005e201058c85a2010514040404c60f06010104d00f05e0e0a44704c60f0e691005040004000e36100204a00b08cd0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ea02d192a39a8cc7a701730073011001020402d19683030193a38cc7b2a57300000193c2b2a57301007473027303830108cdeeac93b1a57304050005000580ade2040100d803d6017300d602b2a4730100d6037302eb027201d195ed93b1a4730393b1db630872027304d804d604db63087202d605b2a5730500d606b2db63087205730600d6077e8c72060206edededededed938cb2720473070001730893c27205d07201938c72060173099272077e730a06927ec172050699997ec1a7069d9c72077e730b067e730c067e720306909c9c7e8cb27204730d0002067e7203067e730e069c9a7207730f9a9c7ec17202067e7310067e9c73117e7312050690b0ada5d90108639593c272087313c1720873147315d90108599a8c7208018c72080273167317",
+                "1" : "0e20003bd19d0187117f130b62e1bcab0939929ff5c7709f843c5c4dd158949285d0",
+                "6" : "0580ade204",
+                "9" : "0580b48913",
+                "2" : "05e201",
+                "7" : "0e201d5afc59838920bb5ef2a8f9d63825a55b1d48e269d7cecee335d637c3ff5f3f",
+                "3" : "05e0e0a447",
+                "8" : "0580ade204",
+                "4" : "058c85a201"
+              }
+            }
+          }
+        ],
+        "dataInputs" : [
+        ],
+        "outputs" : [
+          {
+            "boxId" : "0586c90d0cf6a82dab48c6a79500364ddbd6f81705f5032b03aa287de43dc638",
+            "value" : 94750000,
+            "ergoTree" : "101808cd0279aed8dea2b2a25316d5d49d13bf51c0b2c1dc696974bb4b0c07b5894e998e56040005e0e0a447040404060402040004000e201d5afc59838920bb5ef2a8f9d63825a55b1d48e269d7cecee335d637c3ff5f3f0e20003bd19d0187117f130b62e1bcab0939929ff5c7709f843c5c4dd158949285d005e201058c85a2010514040404c60f06010104d00f05e0e0a44704c60f0e691005040004000e36100204a00b08cd0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ea02d192a39a8cc7a701730073011001020402d19683030193a38cc7b2a57300000193c2b2a57301007473027303830108cdeeac93b1a57304050005000580ade2040100d803d6017300d602b2a4730100d6037302eb027201d195ed93b1a4730393b1db630872027304d804d604db63087202d605b2a5730500d606b2db63087205730600d6077e8c72060206edededededed938cb2720473070001730893c27205d07201938c72060173099272077e730a06927ec172050699997ec1a7069d9c72077e730b067e730c067e720306909c9c7e8cb27204730d0002067e7203067e730e069c9a7207730f9a9c7ec17202067e7310067e9c73117e7312050690b0ada5d90108639593c272087313c1720873147315d90108599a8c7208018c72080273167317",
+            "assets" : [
+            ],
+            "creationHeight" : 693475,
+            "additionalRegisters" : {
+              
+            },
+            "transactionId" : "c8520befd345ff40fcf244b44ffe8cea29c8b116b174cfaf4f2a521604d531a4",
+            "index" : 0
+          },
+          {
+            "boxId" : "4b99c2ef8496a491d176ecaf789d9e1d9aad0c2bf3e70b32e8bad73f48c722b9",
+            "value" : 250000,
+            "ergoTree" : "100e04000500059a0505d00f04020404040608cd03c6543ac8e8059748b1c6209ee419dd49a19ffaf5712a2f34a9412016a3a1d96708cd035b736bebf0c5393f78329f6894af84d1864c7496cc65ddc250ef60cdd75df52008cd021b63e19ab452c84cdc6687242e8494957b1f11e3750c8c184a8425f8a8171d9b05060580ade2040580a8d6b907040ad806d601b2a5730000d602b0a47301d9010241639a8c720201c18c720202d6039d9c730272027303d604b2a5730400d605b2a5730500d606b2a5730600d1968306019683020193c17201720393c27201d073079683020193c17204720393c27204d073089683020193c17205720393c27205d073099683020192c17206999972029c730a7203730b93c2a7c27206927202730c93b1a5730d",
+            "assets" : [
+            ],
+            "creationHeight" : 693475,
+            "additionalRegisters" : {
+              
+            },
+            "transactionId" : "c8520befd345ff40fcf244b44ffe8cea29c8b116b174cfaf4f2a521604d531a4",
+            "index" : 1
+          },
+          {
+            "boxId" : "00ddbeb981c0b08536f72ea41e07a25adbf7bf104ee59b865619a21676e64715",
+            "value" : 5000000,
+            "ergoTree" : "1005040004000e36100204a00b08cd0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ea02d192a39a8cc7a701730073011001020402d19683030193a38cc7b2a57300000193c2b2a57301007473027303830108cdeeac93b1a57304",
+            "assets" : [
+            ],
+            "creationHeight" : 693475,
+            "additionalRegisters" : {
+              
+            },
+            "transactionId" : "c8520befd345ff40fcf244b44ffe8cea29c8b116b174cfaf4f2a521604d531a4",
+            "index" : 2
+          }
+        ],
+        "size" : 1562
+      }
+    "#;
+        let tx: Transaction = serde_json::from_str(tx_json).unwrap();
+        assert_eq!(
+            tx.id(),
+            Digest32::try_from(
+                "c8520befd345ff40fcf244b44ffe8cea29c8b116b174cfaf4f2a521604d531a4".to_string()
+            )
+            .unwrap()
+            .into()
+        );
+    }
 }
diff --git a/ergo-lib/src/chain/json/transaction.rs b/ergo-lib/src/chain/json/transaction.rs
index 7a94e5748..78a55385e 100644
--- a/ergo-lib/src/chain/json/transaction.rs
+++ b/ergo-lib/src/chain/json/transaction.rs
@@ -79,8 +79,10 @@ impl From<Transaction> for TransactionJson {
 #[derive(Error, PartialEq, Eq, Debug, Clone)]
 #[allow(missing_docs)]
 pub enum TransactionFromJsonError {
-    #[error("Tx id parsed from JSON differs from calculated from serialized bytes")]
-    InvalidTxId,
+    #[error(
+        "Tx id parsed from JSON {expected} differs from calculated from serialized bytes {actual}"
+    )]
+    InvalidTxId { expected: TxId, actual: TxId },
     #[error("Tx error: {0}")]
     TransactionError(#[from] TransactionError),
 }
@@ -94,7 +96,11 @@ impl TryFrom<TransactionJson> for Transaction {
         if tx.tx_id == tx_json.tx_id {
             Ok(tx)
         } else {
-            Err(TransactionFromJsonError::InvalidTxId)
+            dbg!(&tx);
+            Err(TransactionFromJsonError::InvalidTxId {
+                expected: tx_json.tx_id,
+                actual: tx.tx_id,
+            })
         }
     }
 }
diff --git a/ergotree-interpreter/src/sigma_protocol/prover/context_extension.rs b/ergotree-interpreter/src/sigma_protocol/prover/context_extension.rs
index b5da657db..09aed3e41 100644
--- a/ergotree-interpreter/src/sigma_protocol/prover/context_extension.rs
+++ b/ergotree-interpreter/src/sigma_protocol/prover/context_extension.rs
@@ -6,7 +6,6 @@ use ergotree_ir::serialization::SigmaParsingError;
 use ergotree_ir::serialization::SigmaSerializable;
 use ergotree_ir::serialization::SigmaSerializeResult;
 use indexmap::IndexMap;
-use std::collections::HashMap;
 use std::convert::TryFrom;
 use thiserror::Error;
 
@@ -29,13 +28,8 @@ impl ContextExtension {
 impl SigmaSerializable for ContextExtension {
     fn sigma_serialize<W: SigmaByteWrite>(&self, w: &mut W) -> SigmaSerializeResult {
         w.put_u8(self.values.len() as u8)?;
-        let mut sorted_values: Vec<(&u8, &Constant)> = self.values.iter().collect();
-        // stable order is important for tx id generation
-        // since JSON encoding does not preserve the order, JSON roundtrip would result in different order
-        // of values and thus a different tx id
-        // see https://github.com/ScorexFoundation/sigmastate-interpreter/issues/681
-        sorted_values.sort_by_key(|(k, _)| *k);
-        sorted_values.iter().try_for_each(|(idx, c)| {
+        let values: Vec<(&u8, &Constant)> = self.values.iter().collect();
+        values.iter().try_for_each(|(idx, c)| {
             w.put_u8(**idx)?;
             c.sigma_serialize(w)
         })?;
@@ -59,9 +53,9 @@ impl SigmaSerializable for ContextExtension {
 pub struct ConstantParsingError(pub String);
 
 // for JSON encoding in ergo-lib
-impl TryFrom<HashMap<String, String>> for ContextExtension {
+impl TryFrom<IndexMap<String, String>> for ContextExtension {
     type Error = ConstantParsingError;
-    fn try_from(values_str: HashMap<String, String>) -> Result<Self, Self::Error> {
+    fn try_from(values_str: IndexMap<String, String>) -> Result<Self, Self::Error> {
         let values = values_str.iter().try_fold(
             IndexMap::with_capacity(values_str.len()),
             |mut acc, pair| {
diff --git a/ergotree-ir/src/chain/json/ergo_box.rs b/ergotree-ir/src/chain/json/ergo_box.rs
index 673e57b9b..9e8a29433 100644
--- a/ergotree-ir/src/chain/json/ergo_box.rs
+++ b/ergotree-ir/src/chain/json/ergo_box.rs
@@ -86,6 +86,7 @@ impl TryFrom<ErgoBoxJson> for ErgoBox {
                 if ergo_box.box_id() == box_id {
                     Ok(ergo_box)
                 } else {
+                    dbg!(&ergo_box);
                     Err(ErgoBoxFromJsonError::InvalidBoxId {
                         json: box_id,
                         actual: ergo_box.box_id(),
diff --git a/ergotree-ir/src/chain/tx_id.rs b/ergotree-ir/src/chain/tx_id.rs
index be2084679..339e7c4d1 100644
--- a/ergotree-ir/src/chain/tx_id.rs
+++ b/ergotree-ir/src/chain/tx_id.rs
@@ -1,6 +1,7 @@
 //! Transaction id type
 use derive_more::Display;
 
+use derive_more::From;
 use ergo_chain_types::Digest32;
 use sigma_ser::ScorexSerializable;
 
@@ -11,7 +12,7 @@ use crate::serialization::SigmaSerializable;
 use crate::serialization::SigmaSerializeResult;
 
 /// Transaction id (ModifierId in sigmastate)
-#[derive(PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Copy, Clone, Display)]
+#[derive(PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Copy, Clone, Display, From)]
 #[cfg_attr(feature = "arbitrary", derive(proptest_derive::Arbitrary))]
 #[cfg_attr(feature = "json", derive(serde::Serialize, serde::Deserialize))]
 pub struct TxId(pub Digest32);


Test patch: