Repository: microsoft/electionguard-python

Issue:
Fix Serialization Issues
## Bug Report

**Expected Behavior**
The serialization behavior of manifests and ballots between C++ and Python is inconsistent.
- Dates are handled inconsistently
- hashing of numbers is handled inconsistently

**Actual Behavior**
_A clear and concise description of the actual behavior._




Discussion:

Patch:
diff --git a/Makefile b/Makefile
index da78882..b73ef59 100644
--- a/Makefile
+++ b/Makefile
@@ -11,8 +11,10 @@ all: environment install build validate auto-lint coverage
 environment:
 	@echo 🔧 ENVIRONMENT SETUP
 	make install-gmp
-	make install-poetry
-	make install-mkdocs
+	#make install-poetry
+	#make install-mkdocs
+	pip install 'poetry==1.0.10'
+	poetry install
 	@echo 🚨 Be sure to add poetry to PATH
 
 install:
@@ -83,7 +85,7 @@ lint:
 	poetry build
 	poetry run twine check dist/*
 	@echo 5.Documentation
-	mkdocs build --strict
+	poetry run mkdocs build --strict
 
 auto-lint:
 	@echo 💚 AUTO LINT
diff --git a/src/electionguard/chaum_pedersen.py b/src/electionguard/chaum_pedersen.py
index 3afce99..7e774f2 100644
--- a/src/electionguard/chaum_pedersen.py
+++ b/src/electionguard/chaum_pedersen.py
@@ -21,7 +21,7 @@ from .nonces import Nonces
 from .proof import Proof, ProofUsage
 
 
-@dataclass(frozen=True)
+@dataclass
 class DisjunctiveChaumPedersenProof(Proof):
     """
     Representation of disjunctive Chaum Pederson proof
@@ -137,7 +137,7 @@ class DisjunctiveChaumPedersenProof(Proof):
         return success
 
 
-@dataclass(frozen=True)
+@dataclass
 class ChaumPedersenProof(Proof):
     """
     Representation of a generic Chaum-Pedersen Zero Knowledge proof
@@ -253,7 +253,7 @@ class ChaumPedersenProof(Proof):
         return success
 
 
-@dataclass(frozen=True)
+@dataclass
 class ConstantChaumPedersenProof(Proof):
     """
     Representation of constant Chaum Pederson proof
diff --git a/src/electionguard/election.py b/src/electionguard/election.py
index b111f45..d19d7de 100644
--- a/src/electionguard/election.py
+++ b/src/electionguard/election.py
@@ -1,6 +1,15 @@
 from dataclasses import dataclass
 
-from .group import Q, P, R, G, ElementModQ, ElementModP
+from .group import (
+    Q,
+    P,
+    R,
+    G,
+    ElementModQ,
+    ElementModP,
+    int_to_p_unchecked,
+    int_to_q_unchecked,
+)
 from .hash import hash_elems
 from .serializable import Serializable
 
@@ -98,7 +107,12 @@ def make_ciphertext_election_context(
     # form the basis of subsequent hash computations.
 
     crypto_base_hash = hash_elems(
-        P, Q, G, number_of_guardians, quorum, description_hash
+        int_to_p_unchecked(P),
+        int_to_q_unchecked(Q),
+        int_to_p_unchecked(G),
+        number_of_guardians,
+        quorum,
+        description_hash,
     )
     crypto_extended_base_hash = hash_elems(crypto_base_hash, commitment_hash)
     return CiphertextElectionContext(
diff --git a/src/electionguard/elgamal.py b/src/electionguard/elgamal.py
index 82be11a..1c966a1 100644
--- a/src/electionguard/elgamal.py
+++ b/src/electionguard/elgamal.py
@@ -10,12 +10,12 @@ from .group import (
     pow_p,
     ZERO_MOD_Q,
     TWO_MOD_Q,
-    int_to_q,
     rand_range_q,
+    int_to_q_unchecked,
 )
 from .hash import hash_elems
-from .logs import log_error
-from .utils import flatmap_optional, get_optional
+from .logs import log_info, log_error
+from .utils import get_optional
 
 ELGAMAL_SECRET_KEY = ElementModQ
 ELGAMAL_PUBLIC_KEY = ElementModP
@@ -128,18 +128,22 @@ def elgamal_encrypt(
     :param m: Message to elgamal_encrypt; must be an integer in [0,Q).
     :param nonce: Randomly chosen nonce in [1,Q).
     :param public_key: ElGamal public key.
-    :return: A ciphertext tuple.
+    :return: An `ElGamalCiphertext`.
     """
     if nonce == ZERO_MOD_Q:
         log_error("ElGamal encryption requires a non-zero nonce")
         return None
 
-    return flatmap_optional(
-        int_to_q(m),
-        lambda e: ElGamalCiphertext(
-            g_pow_p(nonce), mult_p(g_pow_p(e), pow_p(public_key, nonce))
-        ),
-    )
+    pad = g_pow_p(nonce)
+    gpowp_m = g_pow_p(int_to_q_unchecked(m))
+    pubkey_pow_n = pow_p(public_key, nonce)
+    data = mult_p(gpowp_m, pubkey_pow_n)
+
+    log_info(f": publicKey: {public_key.to_hex()}")
+    log_info(f": pad: {pad.to_hex()}")
+    log_info(f": data: {data.to_hex()}")
+
+    return ElGamalCiphertext(pad, data)
 
 
 def elgamal_add(*ciphertexts: ElGamalCiphertext) -> ElGamalCiphertext:
diff --git a/src/electionguard/encrypt.py b/src/electionguard/encrypt.py
index 36d4293..b77c36a 100644
--- a/src/electionguard/encrypt.py
+++ b/src/electionguard/encrypt.py
@@ -1,3 +1,4 @@
+from datetime import datetime
 from typing import List, Optional
 from uuid import getnode
 
@@ -17,7 +18,7 @@ from .ballot_code import get_hash_for_device
 from .election import CiphertextElectionContext
 from .elgamal import elgamal_encrypt
 from .group import ElementModP, ElementModQ, rand_q
-from .logs import log_warning
+from .logs import log_info, log_warning
 from .manifest import (
     InternalManifest,
     ContestDescription,
@@ -58,6 +59,8 @@ class EncryptionDevice(Serializable):
         self.launch_code = launch_code
         self.location = location
 
+        log_info(f": EncryptionDevice: Created: UUID: {uuid} at: {location}")
+
     def get_hash(self) -> ElementModQ:
         """
         Get hash for encryption device
@@ -67,8 +70,12 @@ class EncryptionDevice(Serializable):
             self.uuid, self.session_id, self.launch_code, self.location
         )
 
+    # pylint: disable=no-self-use
     def get_timestamp(self) -> int:
-        pass
+        """
+        Get the current timestamp in utc
+        """
+        return int(datetime.utcnow().timestamp())
 
 
 class EncryptionMediator:
@@ -79,7 +86,7 @@ class EncryptionMediator:
     """
 
     _internal_manifest: InternalManifest
-    _encryption: CiphertextElectionContext
+    _context: CiphertextElectionContext
     _encryption_seed: ElementModQ
 
     def __init__(
@@ -89,15 +96,17 @@ class EncryptionMediator:
         encryption_device: EncryptionDevice,
     ):
         self._internal_manifest = internal_manifest
-        self._encryption = context
+        self._context = context
         self._encryption_seed = encryption_device.get_hash()
 
     def encrypt(self, ballot: PlaintextBallot) -> Optional[CiphertextBallot]:
         """
         Encrypt the specified ballot using the cached election context.
         """
+
+        log_info(f" encrypt: objectId: {ballot.object_id}")
         encrypted_ballot = encrypt_ballot(
-            ballot, self._internal_manifest, self._encryption, self._encryption_seed
+            ballot, self._internal_manifest, self._context, self._encryption_seed
         )
         if encrypted_ballot is not None and encrypted_ballot.code is not None:
             self._encryption_seed = encrypted_ballot.code
@@ -185,6 +194,10 @@ def encrypt_selection(
     selection_nonce = nonce_sequence[selection_description.sequence_order]
     disjunctive_chaum_pedersen_nonce = next(iter(nonce_sequence))
 
+    log_info(
+        f": encrypt_selection: for {selection_description.object_id} hash: {selection_description_hash.to_hex()}"
+    )
+
     selection_representation = selection.vote
 
     # Generate the encryption
@@ -442,6 +455,9 @@ def encrypt_ballot(
         random_master_nonce,
     )
 
+    log_info(f": manifest_hash : {internal_manifest.manifest_hash.to_hex()}")
+    log_info(f": encryption_seed : {encryption_seed.to_hex()}")
+
     encrypted_contests = encrypt_ballot_contests(
         ballot, internal_manifest, context, nonce_seed
     )
diff --git a/src/electionguard/guardian.py b/src/electionguard/guardian.py
index 1c2a907..9555d90 100644
--- a/src/electionguard/guardian.py
+++ b/src/electionguard/guardian.py
@@ -97,7 +97,7 @@ def publish_guardian_record(election_public_key: ElectionPublicKey) -> GuardianR
     )
 
 
-@dataclass(frozen=True)
+@dataclass
 class PrivateGuardianRecord(Serializable):
     """Unpublishable private record containing information per Guardian"""
 
diff --git a/src/electionguard/logs.py b/src/electionguard/logs.py
index c498265..09132d5 100644
--- a/src/electionguard/logs.py
+++ b/src/electionguard/logs.py
@@ -98,7 +98,7 @@ def get_stream_handler() -> logging.StreamHandler:
     Get a Stream Handler, sends only warnings and errors to stdout.
     """
     stream_handler = logging.StreamHandler(sys.stdout)
-    stream_handler.setLevel(logging.WARNING)
+    stream_handler.setLevel(logging.INFO)
     stream_handler.setFormatter(logging.Formatter(FORMAT))
     return stream_handler
 
diff --git a/src/electionguard/manifest.py b/src/electionguard/manifest.py
index b3c92c6..94b85c6 100644
--- a/src/electionguard/manifest.py
+++ b/src/electionguard/manifest.py
@@ -7,9 +7,9 @@ from .ballot import _list_eq
 from .election_object_base import ElectionObjectBase
 from .group import ElementModQ
 from .hash import CryptoHashable, hash_elems
-from .logs import log_warning
+from .logs import log_warning, log_debug
 from .serializable import Serializable
-from .utils import get_optional, to_ticks
+from .utils import get_optional, to_iso_date_string
 
 
 @unique
@@ -103,7 +103,9 @@ class AnnotatedString(Serializable, CryptoHashable):
         """
         A hash representation of the object
         """
-        return hash_elems(self.annotation, self.value)
+        hash = hash_elems(self.annotation, self.value)
+        log_debug(f"{self.__class__} : crypto_hash: {hash.to_hex()}")
+        return hash
 
 
 @dataclass(eq=True, unsafe_hash=True)
@@ -120,7 +122,9 @@ class Language(Serializable, CryptoHashable):
         """
         A hash representation of the object
         """
-        return hash_elems(self.value, self.language)
+        hash = hash_elems(self.value, self.language)
+        log_debug(f"{self.__class__} : crypto_hash: {hash.to_hex()}")
+        return hash
 
 
 @dataclass(eq=True, unsafe_hash=True)
@@ -136,7 +140,9 @@ class InternationalizedText(Serializable, CryptoHashable):
         """
         A hash representation of the object
         """
-        return hash_elems(self.text)
+        hash = hash_elems(self.text)
+        log_debug(f"{self.__class__} : crypto_hash: {hash.to_hex()}")
+        return hash
 
 
 @dataclass(eq=True, unsafe_hash=True)
@@ -155,7 +161,9 @@ class ContactInformation(Serializable, CryptoHashable):
         """
         A hash representation of the object
         """
-        return hash_elems(self.name, self.address_line, self.email, self.phone)
+        hash = hash_elems(self.name, self.address_line, self.email, self.phone)
+        log_debug(f"{self.__class__} : crypto_hash: {hash.to_hex()}")
+        return hash
 
 
 @dataclass(eq=True, unsafe_hash=True)
@@ -174,9 +182,11 @@ class GeopoliticalUnit(ElectionObjectBase, CryptoHashable):
         """
         A hash representation of the object
         """
-        return hash_elems(
+        hash = hash_elems(
             self.object_id, self.name, str(self.type.name), self.contact_information
         )
+        log_debug(f"{self.__class__} : crypto_hash: {hash.to_hex()}")
+        return hash
 
 
 @dataclass(eq=True, unsafe_hash=True)
@@ -193,9 +203,11 @@ class BallotStyle(ElectionObjectBase, CryptoHashable):
         """
         A hash representation of the object
         """
-        return hash_elems(
+        hash = hash_elems(
             self.object_id, self.geopolitical_unit_ids, self.party_ids, self.image_uri
         )
+        log_debug(f"{self.__class__} : crypto_hash: {hash.to_hex()}")
+        return hash
 
 
 @dataclass(eq=True, unsafe_hash=True)
@@ -220,13 +232,15 @@ class Party(ElectionObjectBase, CryptoHashable):
         """
         A hash representation of the object
         """
-        return hash_elems(
+        hash = hash_elems(
             self.object_id,
             self.name,
             self.abbreviation,
             self.color,
             self.logo_uri,
         )
+        log_debug(f"{self.__class__} : crypto_hash: {hash.to_hex()}")
+        return hash
 
 
 @dataclass(eq=True, unsafe_hash=True)
@@ -256,7 +270,9 @@ class Candidate(ElectionObjectBase, CryptoHashable):
         """
         A hash representation of the object
         """
-        return hash_elems(self.object_id, self.name, self.party_id, self.image_uri)
+        hash = hash_elems(self.object_id, self.name, self.party_id, self.image_uri)
+        log_debug(f"{self.__class__} : crypto_hash: {hash.to_hex()}")
+        return hash
 
 
 @dataclass(eq=True, unsafe_hash=True)
@@ -287,7 +303,9 @@ class SelectionDescription(ElectionObjectBase, CryptoHashable):
         """
         A hash representation of the object
         """
-        return hash_elems(self.object_id, self.sequence_order, self.candidate_id)
+        hash = hash_elems(self.object_id, self.sequence_order, self.candidate_id)
+        log_debug(f"{self.__class__} : crypto_hash: {hash.to_hex()}")
+        return hash
 
 
 # pylint: disable=too-many-instance-attributes
@@ -358,7 +376,7 @@ class ContestDescription(ElectionObjectBase, CryptoHashable):
         description match up.
         """
         # remove any placeholders from the hash mechanism
-        return hash_elems(
+        hash = hash_elems(
             self.object_id,
             self.sequence_order,
             self.electoral_district_id,
@@ -370,6 +388,8 @@ class ContestDescription(ElectionObjectBase, CryptoHashable):
             self.votes_allowed,
             self.ballot_selections,
         )
+        log_debug(f"{self.__class__} : crypto_hash: {hash.to_hex()}")
+        return hash
 
     def is_valid(self) -> bool:
         """
@@ -561,12 +581,11 @@ class Manifest(Serializable, CryptoHashable):
         """
         Returns a hash of the metadata components of the election
         """
-
-        return hash_elems(
+        hash = hash_elems(
             self.election_scope_id,
             str(self.type.name),
-            to_ticks(self.start_date),
-            to_ticks(self.end_date),
+            to_iso_date_string(self.start_date),
+            to_iso_date_string(self.end_date),
             self.name,
             self.contact_information,
             self.geopolitical_units,
@@ -574,6 +593,8 @@ class Manifest(Serializable, CryptoHashable):
             self.contests,
             self.ballot_styles,
         )
+        log_debug(f"{self.__class__} : crypto_hash: {hash.to_hex()}")
+        return hash
 
     def is_valid(self) -> bool:
         """
diff --git a/src/electionguard/schnorr.py b/src/electionguard/schnorr.py
index 1eb9df3..bd3a12d 100644
--- a/src/electionguard/schnorr.py
+++ b/src/electionguard/schnorr.py
@@ -14,7 +14,7 @@ from .logs import log_warning
 from .proof import Proof, ProofUsage
 
 
-@dataclass(frozen=True)
+@dataclass
 class SchnorrProof(Proof):
     """
     Representation of a Schnorr proof
diff --git a/src/electionguard/utils.py b/src/electionguard/utils.py
index 33c7e88..0f24292 100644
--- a/src/electionguard/utils.py
+++ b/src/electionguard/utils.py
@@ -78,6 +78,21 @@ def to_ticks(date_time: datetime) -> int:
     return int(ticks)
 
 
+def to_iso_date_string(date_time: datetime) -> str:
+    """
+    Return the number of ticks for a date time.
+    Ticks are defined here as number of seconds since the unix epoch (00:00:00 UTC on 1 January 1970)
+    :param date_time: Date time to convert
+    :return: number of ticks
+    """
+    utc_datetime = (
+        date_time.astimezone(timezone.utc).replace(microsecond=0)
+        if date_time.tzinfo
+        else date_time.replace(microsecond=0)
+    )
+    return utc_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
 def space_between_capitals(base: str) -> str:
     """
     Return a modified string with spaces between capital letters


Test patch:
diff --git a/tests/property/test_decrypt_with_secrets.py b/tests/property/test_decrypt_with_secrets.py
index 1f8e7e9..5aea0cf 100644
--- a/tests/property/test_decrypt_with_secrets.py
+++ b/tests/property/test_decrypt_with_secrets.py
@@ -488,7 +488,7 @@ class TestDecryptWithSecrets(unittest.TestCase):
         self.assertIsNone(result_from_nonce_seed_tampered)
 
     @settings(
-        deadline=timedelta(milliseconds=2000),
+        deadline=timedelta(milliseconds=5000),
         suppress_health_check=[HealthCheck.too_slow],
         max_examples=1,
         # disabling the "shrink" phase, because it runs very slowly
diff --git a/tests/property/test_encrypt.py b/tests/property/test_encrypt.py
index 266c14e..6feca60 100644
--- a/tests/property/test_encrypt.py
+++ b/tests/property/test_encrypt.py
@@ -23,6 +23,7 @@ from electionguard.elgamal import (
     elgamal_add,
 )
 from electionguard.encrypt import (
+    EncryptionDevice,
     contest_from,
     encrypt_ballot,
     encrypt_contest,
@@ -580,7 +581,7 @@ class TestEncrypt(unittest.TestCase):
             )
         )
 
-    def test_encrypt_ballot_with_stateful_composer_succeeds(self):
+    def test_encrypt_ballot_with_composer_succeeds(self):
         # Arrange
         keypair = elgamal_keypair_from_secret(int_to_q(2))
         manifest = election_factory.get_fake_manifest()
@@ -607,7 +608,7 @@ class TestEncrypt(unittest.TestCase):
             )
         )
 
-    def test_encrypt_simple_ballot_from_files_succeeds(self):
+    def test_encrypt_simple_ballot_from_file_with_composer_succeeds(self):
         # Arrange
         keypair = elgamal_keypair_from_secret(int_to_q(2))
         manifest = election_factory.get_simple_manifest_from_file()
@@ -618,12 +619,14 @@ class TestEncrypt(unittest.TestCase):
         data = ballot_factory.get_simple_ballot_from_file()
         self.assertTrue(data.is_valid(internal_manifest.ballot_styles[0].object_id))
 
-        device = election_factory.get_encryption_device()
+        device = EncryptionDevice(12345, 23456, 34567, "Location")
         subject = EncryptionMediator(internal_manifest, context, device)
 
         # Act
         result = subject.encrypt(data)
 
+        print(result.to_json())
+
         # Assert
         self.assertIsNotNone(result)
         self.assertEqual(data.object_id, result.object_id)
@@ -635,6 +638,36 @@ class TestEncrypt(unittest.TestCase):
             )
         )
 
+    def test_encrypt_simple_ballot_from_files_succeeds(self) -> None:
+        # Arrange
+        keypair = elgamal_keypair_from_secret(int_to_q(2))
+        manifest = election_factory.get_simple_manifest_from_file()
+        internal_manifest, context = election_factory.get_fake_ciphertext_election(
+            manifest, keypair.public_key
+        )
+
+        device = EncryptionDevice(12345, 23456, 34567, "Location")
+        ballot = ballot_factory.get_simple_ballot_from_file()
+        self.assertTrue(ballot.is_valid(internal_manifest.ballot_styles[0].object_id))
+
+        # Act
+
+        ciphertext = encrypt_ballot(
+            ballot, internal_manifest, context, device.get_hash(), TWO_MOD_Q
+        )
+        print(ciphertext.to_json())
+
+        # Assert
+        self.assertIsNotNone(ciphertext)
+        self.assertEqual(ballot.object_id, ciphertext.object_id)
+        self.assertTrue(
+            ciphertext.is_valid_encryption(
+                internal_manifest.manifest_hash,
+                keypair.public_key,
+                context.crypto_extended_base_hash,
+            )
+        )
+
     @settings(
         deadline=timedelta(milliseconds=4000),
         suppress_health_check=[HealthCheck.too_slow],
diff --git a/tests/property/test_hash.py b/tests/property/test_hash.py
index 1ffd765..19f0df7 100644
--- a/tests/property/test_hash.py
+++ b/tests/property/test_hash.py
@@ -1,5 +1,7 @@
 import unittest
 
+from typing import List, Optional
+
 from hypothesis import given
 
 from electionguard.group import ElementModQ
@@ -25,3 +27,46 @@ class TestHash(unittest.TestCase):
             self.assertEqual(ha, hb)
         if ha != hb:
             self.assertNotEqual(a, b)
+
+    def test_hash_value_for_zero_string_different_from_zero_number(self):
+        self.assertNotEqual(hash_elems(0), hash_elems("0"))
+
+    def test_hash_for_non_zero_number_string_same_as_explicit_number(self):
+        self.assertEqual(hash_elems(1), hash_elems("1"))
+
+    def test_different_strings_casing_not_the_same_hash(self):
+        self.assertNotEqual(
+            hash_elems("Welcome To ElectionGuard"),
+            hash_elems("welcome to electionguard"),
+        )
+
+    def test_hash_for_empty_string_same_as_null_string(self):
+        self.assertEqual(hash_elems(""), hash_elems("null"))
+
+    def test_hash_for_none_same_as_null_string(self):
+        self.assertEqual(hash_elems(None), hash_elems("null"))
+
+    def test_hash_for_none_same_as_explicit_zero_number(self):
+        self.assertEqual(hash_elems(None), hash_elems(0))
+
+    def test_hash_of_save_values_in_list_are_same_hash(self):
+        self.assertEqual(hash_elems(["0", "0"]), hash_elems(["0", "0"]))
+
+    def test_hash_empty_list_same_as_hash_null_string(self):
+        null_list: Optional[List[str]] = None
+        empty_list: List[str] = []
+
+        self.assertEqual(hash_elems(null_list), hash_elems(empty_list))
+        self.assertEqual(hash_elems(empty_list), hash_elems(None))
+        self.assertEqual(hash_elems(empty_list), hash_elems(""))
+        self.assertEqual(hash_elems(empty_list), hash_elems("null"))
+
+    def test_hash_value_from_nested_list_and_result_of_hashed_list_by_taking_the_hex(
+        self,
+    ):
+        nested_hash = hash_elems(["0", "1"], "3")
+        non_nested_1 = hash_elems("0", "1")
+        non_nested_2 = hash_elems(non_nested_1.to_hex(), "3")
+
+        self.assertNotEqual(nested_hash, non_nested_1)
+        self.assertEqual(nested_hash, non_nested_2)