Repository: quickwit-oss/tantivy-py
Pull request #401: Updated `pythonize` and `pyo3`
Author: login: jamesbraza
Created: 2024-12-17T04:42:43Z | Merged: 2025-05-05T10:47:43Z

Closes https://github.com/quickwit-oss/tantivy-py/issues/371


Diff:
diff --git a/Cargo.lock b/Cargo.lock
index e6325ae4..44fac58d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -73,12 +73,6 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
 [[package]]
 name = "bitflags"
 version = "2.4.1"
@@ -450,9 +444,9 @@ dependencies = [
 
 [[package]]
 name = "heck"
-version = "0.4.1"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
 name = "htmlescape"
@@ -567,16 +561,6 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
 
-[[package]]
-name = "lock_api"
-version = "0.4.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"
-dependencies = [
- "autocfg",
- "scopeguard",
-]
-
 [[package]]
 name = "log"
 version = "0.4.20"
@@ -690,29 +674,6 @@ dependencies = [
  "stable_deref_trait",
 ]
 
-[[package]]
-name = "parking_lot"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
-dependencies = [
- "lock_api",
- "parking_lot_core",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.9.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_syscall",
- "smallvec",
- "windows-targets 0.48.5",
-]
-
 [[package]]
 name = "pin-project-lite"
 version = "0.2.13"
@@ -770,33 +731,23 @@ dependencies = [
 
 [[package]]
 name = "pyo3"
-version = "0.21.2"
+version = "0.24.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8"
+checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219"
 dependencies = [
  "cfg-if",
  "chrono",
  "indoc",
  "libc",
  "memoffset",
- "parking_lot",
+ "once_cell",
  "portable-atomic",
- "pyo3-build-config 0.21.2",
+ "pyo3-build-config",
  "pyo3-ffi",
  "pyo3-macros",
  "unindent",
 ]
 
-[[package]]
-name = "pyo3-build-config"
-version = "0.21.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50"
-dependencies = [
- "once_cell",
- "target-lexicon 0.12.14",
-]
-
 [[package]]
 name = "pyo3-build-config"
 version = "0.24.2"
@@ -804,24 +755,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999"
 dependencies = [
  "once_cell",
- "target-lexicon 0.13.2",
+ "target-lexicon",
 ]
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.21.2"
+version = "0.24.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403"
+checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33"
 dependencies = [
  "libc",
- "pyo3-build-config 0.21.2",
+ "pyo3-build-config",
 ]
 
 [[package]]
 name = "pyo3-macros"
-version = "0.21.2"
+version = "0.24.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c"
+checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
@@ -831,22 +782,22 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.21.2"
+version = "0.24.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c"
+checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a"
 dependencies = [
  "heck",
  "proc-macro2",
- "pyo3-build-config 0.21.2",
+ "pyo3-build-config",
  "quote",
  "syn",
 ]
 
 [[package]]
 name = "pythonize"
-version = "0.21.1"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d0664248812c38cc55a4ed07f88e4df516ce82604b93b1ffdc041aa77a6cb3c"
+checksum = "d5bcac0d0b71821f0d69e42654f1e15e5c94b85196446c4de9588951a2117e7b"
 dependencies = [
  "pyo3",
  "serde",
@@ -927,15 +878,6 @@ dependencies = [
  "crossbeam-utils",
 ]
 
-[[package]]
-name = "redox_syscall"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
-dependencies = [
- "bitflags 1.3.2",
-]
-
 [[package]]
 name = "regex"
 version = "1.10.2"
@@ -987,7 +929,7 @@ version = "0.38.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3"
 dependencies = [
- "bitflags 2.4.1",
+ "bitflags",
  "errno",
  "libc",
  "linux-raw-sys 0.4.10",
@@ -1000,7 +942,7 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf"
 dependencies = [
- "bitflags 2.4.1",
+ "bitflags",
  "errno",
  "libc",
  "linux-raw-sys 0.9.4",
@@ -1113,7 +1055,7 @@ dependencies = [
  "futures",
  "itertools",
  "pyo3",
- "pyo3-build-config 0.24.2",
+ "pyo3-build-config",
  "pythonize",
  "serde",
  "serde_json",
@@ -1266,12 +1208,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "target-lexicon"
-version = "0.12.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f"
-
 [[package]]
 name = "target-lexicon"
 version = "0.13.2"
@@ -1627,7 +1563,7 @@ version = "0.39.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
 dependencies = [
- "bitflags 2.4.1",
+ "bitflags",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 34db8db6..101c4927 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -19,10 +19,10 @@ chrono = "0.4.40"
 tantivy = "0.24.0"
 itertools = "0.14.0"
 futures = "0.3.31"
-pythonize = "0.21.0"
+pythonize = "0.24.0"
 serde = "1.0"
 serde_json = "1.0.140"
 
 [dependencies.pyo3]
-version = "0.21.0"
+version = "0.24.2"
 features = ["chrono", "extension-module"]
diff --git a/src/document.rs b/src/document.rs
index afd655eb..10d18a1f 100644
--- a/src/document.rs
+++ b/src/document.rs
@@ -52,9 +52,8 @@ pub(crate) fn extract_value(any: &Bound<PyAny>) -> PyResult<Value> {
         return Ok(Value::Bytes(b));
     }
     if let Ok(dict) = any.downcast::<PyDict>() {
-        if let Ok(json_dict) = pythonize::depythonize_bound::<
-            BTreeMap<String, Value>,
-        >(dict.clone().into_any())
+        if let Ok(json_dict) =
+            pythonize::depythonize::<BTreeMap<String, Value>>(&dict.as_ref())
         {
             return Ok(Value::Object(json_dict.into_iter().collect()));
         } else {
@@ -136,8 +135,8 @@ pub(crate) fn extract_value_for_type(
             let dict = any
                 .downcast::<PyDict>()
                 .map_err(to_pyerr_for_type("Json", field_name, any))?;
-            let map = pythonize::depythonize_bound::<BTreeMap<String, Value>>(
-                dict.clone().into_any(),
+            let map = pythonize::depythonize::<BTreeMap<String, Value>>(
+                &dict.as_ref(),
             )?;
             Value::Object(map.into_iter().collect())
         }
@@ -312,6 +311,60 @@ where
     i64::deserialize(deserializer).map(tv::DateTime::from_timestamp_nanos)
 }
 
+fn deserialize_json_object_as_i64<'de, D>(
+    deserializer: D,
+) -> Result<Vec<(String, Value)>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let raw_object = Vec::deserialize(deserializer)?;
+    let converted_object = raw_object
+        .into_iter()
+        .map(|(key, value)| {
+            let converted_value = match value {
+                serde_json::Value::Number(num) => {
+                    if let Some(i) = num.as_i64() {
+                        Value::I64(i)
+                    } else {
+                        Value::F64(num.as_f64().unwrap())
+                    }
+                }
+                serde_json::Value::Object(obj) => {
+                    Value::Object(deserialize_json_object_as_i64_inner(obj))
+                }
+                _ => Value::from(value),
+            };
+            (key, converted_value)
+        })
+        .collect();
+
+    Ok(converted_object)
+}
+
+fn deserialize_json_object_as_i64_inner(
+    raw_object: serde_json::Map<String, serde_json::Value>,
+) -> Vec<(String, Value)> {
+    raw_object
+        .into_iter()
+        .map(|(key, value)| {
+            let converted_value = match value {
+                serde_json::Value::Number(num) => {
+                    if let Some(i) = num.as_i64() {
+                        Value::I64(i)
+                    } else {
+                        Value::F64(num.as_f64().unwrap())
+                    }
+                }
+                serde_json::Value::Object(obj) => {
+                    Value::Object(deserialize_json_object_as_i64_inner(obj))
+                }
+                _ => Value::from(value),
+            };
+            (key, converted_value)
+        })
+        .collect()
+}
+
 /// An equivalent type to [`tantivy::schema::Value`], but unlike the tantivy crate's serialization
 /// implementation, it uses tagging in its serialization and deserialization to differentiate
 /// between different integer types.
@@ -347,6 +400,7 @@ enum SerdeValue {
     /// Array
     Array(Vec<Value>),
     /// Object value.
+    #[serde(deserialize_with = "deserialize_json_object_as_i64")]
     Object(Vec<(String, Value)>),
     /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
     IpAddr(Ipv6Addr),
@@ -581,6 +635,7 @@ impl Document {
     }
 
     #[staticmethod]
+    #[pyo3(signature = (py_dict, schema=None))]
     fn from_dict(
         py_dict: &Bound<PyDict>,
         schema: Option<&Schema>,
@@ -719,9 +774,7 @@ impl Document {
                 serde_json::from_str(json_str).map_err(to_pyerr)?;
             self.add_value(field_name, json_map);
             Ok(())
-        } else if let Ok(json_map) =
-            pythonize::depythonize_bound::<JsonMap>(value.clone())
-        {
+        } else if let Ok(json_map) = pythonize::depythonize::<JsonMap>(value) {
             self.add_value(field_name, json_map);
             Ok(())
         } else {
@@ -826,7 +879,7 @@ impl Document {
 
     #[staticmethod]
     fn _internal_from_pythonized(serialized: &Bound<PyAny>) -> PyResult<Self> {
-        pythonize::depythonize_bound(serialized.clone()).map_err(to_pyerr)
+        pythonize::depythonize(&serialized).map_err(to_pyerr)
     }
 
     fn __reduce__<'a>(
diff --git a/src/query.rs b/src/query.rs
index 110f75b7..20c55740 100644
--- a/src/query.rs
+++ b/src/query.rs
@@ -15,10 +15,8 @@ use tantivy as tv;
 struct OccurQueryPair(Occur, Query);
 
 impl<'source> FromPyObject<'source> for OccurQueryPair {
-    fn extract(ob: &'source PyAny) -> PyResult<Self> {
-        let tuple = ob.downcast::<PyTuple>()?;
-        let occur = tuple.get_item(0)?.extract()?;
-        let query = tuple.get_item(1)?.extract()?;
+    fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult<Self> {
+        let (occur, query): (Occur, Query) = ob.extract()?;
 
         Ok(OccurQueryPair(occur, query))
     }
@@ -223,6 +221,7 @@ impl Query {
 
     /// Construct a Tantivy's DisjunctionMaxQuery
     #[staticmethod]
+    #[pyo3(signature = (subqueries, tie_breaker=None))]
     pub(crate) fn disjunction_max_query(
         subqueries: Vec<Query>,
         tie_breaker: Option<Bound<PyFloat>>,
diff --git a/src/schema.rs b/src/schema.rs
index f18f84e3..4d7bb997 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -63,7 +63,7 @@ impl Schema {
 
     #[staticmethod]
     fn _internal_from_pythonized(serialized: &Bound<PyAny>) -> PyResult<Self> {
-        pythonize::depythonize_bound(serialized.clone()).map_err(to_pyerr)
+        pythonize::depythonize(&serialized).map_err(to_pyerr)
     }
 
     fn __reduce__<'a>(


Reviews:
user: login: copilot-pull-request-reviewer[bot]
body: Copilot reviewed 2 out of 2 changed files in this pull request and generated no comments.



state: COMMENTED
author_association: NONE
submitted_at: 2024-12-17T06:51:36Z
user: login: copilot-pull-request-reviewer[bot]
body: Copilot reviewed 3 out of 3 changed files in this pull request and generated no comments.



state: COMMENTED
author_association: NONE
submitted_at: 2024-12-17T11:10:37Z
user: login: wallies
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-04T12:55:12Z
user: login: wallies
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-04T12:55:47Z
user: login: jamesbraza
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-04T18:05:38Z
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This PR updates the pythonize usage and pyo3 API calls, addressing dependency upgrades and refining JSON deserialization for documents. Key changes include:
- Replacing pythonize::depythonize_bound with pythonize::depythonize in multiple modules.
- Renaming and refactoring extraction functions to improve API clarity.
- Introducing custom JSON deserialization functions for converting numeric values.

### Reviewed Changes

Copilot reviewed 5 out of 5 changed files in this pull request and generated 1 comment.

| File            | Description                                                            |
| --------------- | ---------------------------------------------------------------------- |
| src/schema.rs   | Updated pythonize usage to remove unnecessary cloning.                 |
| src/query.rs    | Refactored tuple extraction with a renamed function for clarity.       |
| src/document.rs | Revised pythonize conversion and added custom JSON deserialization.    |
| Cargo.toml      | Upgraded dependency versions for pythonize and pyo3.                   |


<details>
<summary>Comments suppressed due to low confidence (2)</summary>

**src/query.rs:19**
* [nitpick] The function renaming to 'extract_bound' and the destructuring of the tuple improves clarity; ensure that this naming aligns with the rest of the codebase conventions to avoid confusion.
```
let (occur, query): (Occur, Query) = ob.extract()?
```
**src/schema.rs:66**
* The removal of the clone indicates better performance, but please verify that passing a reference to `depythonize` correctly handles the Bound<PyAny> without unintended side effects.
```
pythonize::depythonize(&serialized).map_err(to_pyerr)
```
</details>


state: COMMENTED
author_association: NONE
submitted_at: 2025-05-04T21:51:08Z

Review comments:
diff_hunk: @@ -19,10 +19,10 @@ chrono = "0.4.40"
 tantivy = "0.24.0"
 itertools = "0.14.0"
 futures = "0.3.31"
-pythonize = "0.21.0"
+pythonize = "0.23.0"
 serde = "1.0"
 serde_json = "1.0.140"
 
 [dependencies.pyo3]
-version = "0.21.0"
+version = "0.23.5"
path: Cargo.toml
user: login: wallies
body: Should we use the same version as pyo3 build config above 0.24.2?
created_at: 2025-05-04T12:55:11Z
author_association: COLLABORATOR
diff_hunk: @@ -19,10 +19,10 @@ chrono = "0.4.40"
 tantivy = "0.24.0"
 itertools = "0.14.0"
 futures = "0.3.31"
-pythonize = "0.21.0"
+pythonize = "0.23.0"
 serde = "1.0"
 serde_json = "1.0.140"
 
 [dependencies.pyo3]
-version = "0.21.0"
+version = "0.23.5"
path: Cargo.toml
user: login: jamesbraza
body: Just took care of this, and went ahead and `rebase`'d atop `master` too
created_at: 2025-05-04T18:05:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -312,6 +311,60 @@ where
     i64::deserialize(deserializer).map(tv::DateTime::from_timestamp_nanos)
 }
 
+fn deserialize_json_object_as_i64<'de, D>(
+    deserializer: D,
+) -> Result<Vec<(String, Value)>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let raw_object = Vec::deserialize(deserializer)?;
+    let converted_object = raw_object
+        .into_iter()
+        .map(|(key, value)| {
+            let converted_value = match value {
+                serde_json::Value::Number(num) => {
+                    if let Some(i) = num.as_i64() {
+                        Value::I64(i)
+                    } else {
+                        Value::F64(num.as_f64().unwrap())
path: src/document.rs
user: login: Copilot
body: Using unwrap on 'num.as_f64()' might cause a panic if the conversion fails; consider handling the potential None case with proper error handling.
```suggestion
                    } else if let Some(f) = num.as_f64() {
                        Value::F64(f)
                    } else {
                        return Err(serde::de::Error::custom("Failed to convert number to f64"));
```
created_at: 2025-05-04T21:51:08Z
author_association: NONE

PR comments:
user: login: cjrh
created_at: 2025-01-08T13:56:25Z
body: The reason the pickle test fails is because of something that is different in the `pythonize` package in the new version.  When a `Document` object is deserialized, values in the json object that were `i64`, become `u64` after deserialization:

```
Comparing:

{"birth": [Date(2019-08-12T13:00:05Z)], "bytes": [Bytes([97, 98, 99])], "facet": [Facet(Facet(/europe/france))], "float": [F64(1.0)], "integer": [I64(5)], "json": [Object({"a": I64(1), "b": I64(-2)})], "title": [Str("hello world!")], "unsigned": [U64(1)]}

{"birth": [Date(2019-08-12T13:00:05Z)], "bytes": [Bytes([97, 98, 99])], "facet": [Facet(Facet(/europe/france))], "float": [F64(1.0)], "integer": [I64(5)], "json": [Object({"a": U64(1), "b": I64(-2)})], "title": [Str("hello world!")], "unsigned": [U64(1)]}
```

Look carefully at the types in the json object. Before serialization, `I64(1)`, but after deserialization, `U64`.  This seems to only happen with the JSON object. If that is commented (`.add_json`) in the test then the test passes.
author_association: COLLABORATOR
user: login: jamesbraza
created_at: 2025-02-17T22:11:49Z
body: Yeah I concur with your breakdown that the serialization/deserialization cycle is considering the JSON's `a` key to be unsigned now.

> If that is commented (`.add_json`) in the test then the test passes.

Fwiw, going to `assert orig.to_dict() == pickled.to_dict()` leads to the test passing as well. So it's something within `__eq__` specifically.

---

How are you getting that printed representation with `I64`/`U64`? Using `repr` or `to_dict` I don't see those values.

Also, `Document` has no `__eq__` method, so I am not yet understanding how the `__eq__` check is actually failing.
author_association: CONTRIBUTOR
user: login: cjrh
created_at: 2025-02-17T23:15:10Z
body: If the `assert .to_dict()` change makes the test pass, I am ok to proceed with that. On the python side there is no difference between I64 and u64, nor in a json representation of those payloads if they ever get serialized for anything.  What do you reckon? 
author_association: COLLABORATOR
user: login: jamesbraza
created_at: 2025-02-17T23:22:00Z
body: I think it's important for `__eq__` to work across serialization/deserialization.

What do you think of implementing a `Document.__eq__` to invoke `to_dict`:

```python
class Document:
    ...

    def __eq__(self, other) -> bool:
        if not isinstance(other, type(self)):
            return NotImplemented
        return self.to_dict() == other.to_dict()
```

Also, it may be preferred to do this in `document.rs` with the Rust API, as opposed to in Python only.
author_association: CONTRIBUTOR
user: login: cjrh
created_at: 2025-02-17T23:24:36Z
body: Overriding `__eq__` makes me nervous.

> I think it's important for __eq__ to work across serialization/deserialization.

Good point. Perhaps we should make a reproducer of the issue using only pythonize, and ask from help from that project.
author_association: COLLABORATOR
user: login: jamesbraza
created_at: 2025-02-18T00:12:42Z
body: I would be happy to do so, if you can point out how you printed the `Document` such that `U64`/`I64` stuff showed up.

Also, just so I can understand, how do you know the issue is with `pythonize` vs `pyo3`?
author_association: CONTRIBUTOR
user: login: cjrh
created_at: 2025-02-18T01:46:05Z
body: > if you can point out how you printed the Document such that U64/I64 stuff showed up.

From memory I put print statements somewhere. Hopefully I still have that code somewhere. I will check tomorrow.
author_association: COLLABORATOR
user: login: cjrh
created_at: 2025-02-18T10:11:45Z
body: To print out that data, I added a print inside `__richcmp__` like this:

```rust
impl Document {
    fn __richcmp__<'py>(
        &self,
        other: &Self,
        op: CompareOp,
        py: Python<'py>
    ) -> PyResult<Bound<'py, PyAny>> {
        println!("\n\n\nComparing:\n\n{:?}\n\n{:?}", self.field_values, other.field_values);
        match op {
            CompareOp::Eq => {
                let v = (self == other).into_pyobject(py)?.to_owned().into_any();
                Ok(v)
            },
            CompareOp::Ne => {
                let v = (self != other).into_pyobject(py)?.to_owned().into_any();
                Ok(v)
            },
            _ => {
                let v = PyNotImplemented::get(py).to_owned().into_any();
                Ok(v)
            }
        }
    }
}
```

Tracing backwards from here, I also added some prints in the `_internal_from_pythonized()` method:

```rust
impl Document {
    #[staticmethod]
    fn _internal_from_pythonized(serialized: &Bound<PyAny>) -> PyResult<Self> {
        println!("\n\n\nDeserializing: {:?}", serialized);
        let out = pythonize::depythonize(serialized).map_err(to_pyerr);
        let out: Document = out.unwrap();
        println!("\n\n\nDeserialized: {:?}", out);
        println!("\n\n\nDeserialized: {:?}", out.field_values);
        Ok(out)
    }
}
```

So when I run the tests I get this stdout:

```
------------------------------------- Captured stdout call -------------------------------------



Deserializing: {'birth': [{'Date': 1565614805000000000}], 'bytes': [{'Bytes': [97, 98, 99]}], 'facet': [{'Facet': '/europe/france'}], 'float': [{'F64': 1.0}], 'integer': [{'I64': 5}], 'json': [{'Object': {'a': 1, 'b': 2}}], 'title': [{'Str': 'hello world!'}], 'unsigned': [{'U64': 1}]}



Deserialized: Document(birth=[2019-08-12],bytes=[[97, 98, 9],facet=[/europe/fr],float=[1],integer=[5],json=[{"a":1,"b"],title=[hello worl],unsigned=[1])



Deserialized: {"birth": [Date(2019-08-12T13:00:05Z)], "bytes": [Bytes([97, 98, 99])], "facet": [Facet(Facet(/europe/france))], "float": [F64(1.0)], "integer": [I64(5)], "json": [Object({"a": U64(1), "b": U64(2)})], "title": [Str("hello world!")], "unsigned": [U64(1)]}
orig: Document(birth=[2019-08-12],bytes=[[97, 98, 9],facet=[/europe/fr],float=[1],integer=[5],json=[{"a":1,"b"],title=[hello worl],unsigned=[1])
pickled: Document(birth=[2019-08-12],bytes=[[97, 98, 9],facet=[/europe/fr],float=[1],integer=[5],json=[{"a":1,"b"],title=[hello worl],unsigned=[1])



Comparing:

{"birth": [Date(2019-08-12T13:00:05Z)], "bytes": [Bytes([97, 98, 99])], "facet": [Facet(Facet(/europe/france))], "float": [F64(1.0)], "integer": [I64(5)], "json": [Object({"a": I64(1), "b": I64(2)})], "title": [Str("hello world!")], "unsigned": [U64(1)]}

{"birth": [Date(2019-08-12T13:00:05Z)], "bytes": [Bytes([97, 98, 99])], "facet": [Facet(Facet(/europe/france))], "float": [F64(1.0)], "integer": [I64(5)], "json": [Object({"a": U64(1), "b": U64(2)})], "title": [Str("hello world!")], "unsigned": [U64(1)]}
```
author_association: COLLABORATOR
user: login: jamesbraza
created_at: 2025-03-02T23:12:06Z
body: Okay I made https://github.com/jamesbraza/tantivy-py/pull/1, which:

1. PR description states my current understanding of the problem (newer `pythonize` more aggressively making U64)
2. Poses a somewhat hacky solution, basically checking if one can convert back to I64

I am not happy with this, but I am really in over my head here, so I made https://github.com/davidhewitt/pythonize/issues/80 basically as a SOS.
author_association: CONTRIBUTOR
user: login: jamesbraza
created_at: 2025-03-02T23:42:50Z
body: Perhaps `extract_value` at https://github.com/quickwit-oss/tantivy-py/blob/0.22.0/src/document.rs#L37-L39 has something to do with it? I don't see a U64 case, more just talking out loud
author_association: CONTRIBUTOR
user: login: cjrh
created_at: 2025-03-03T12:27:28Z
body: @jamesbraza It would be great if you could add a very short reproducer to your issue on pythonize, so that David can easily run it and see the change in type during roundtrip.
author_association: COLLABORATOR
user: login: cjrh
created_at: 2025-03-03T12:28:41Z
body: and

> but I am really in over my head here

Don't worry, we're all learning all the time. Thanks for your continued interest in helping here ❤️ 
author_association: COLLABORATOR
user: login: cjrh
created_at: 2025-05-04T11:04:58Z
body: I pushed a commit with a new strategy: for the JSON type, deserialize all integers to I64. This solves our problem with the pickle test, and without breaking any handling for toplevel I64/U64 integers. This is probably the best we can do here.

I will also try to update this branch on top of current master.
author_association: COLLABORATOR
user: login: cjrh
created_at: 2025-05-04T12:10:58Z
body: Successfully merged the main branch in.
author_association: COLLABORATOR
user: login: jamesbraza
created_at: 2025-05-04T19:25:13Z
body: My apologies for the two month hiatus here, good to see you guys pushed this forward.

To catch myself up and concrete my understanding, just giving a tl;dr on this PR's story:

1. With newer `pythonize`, we found I64 objects after deserialization become U64, breaking `Document.__eq__`.
2. This happens is because newer `pythonize::depythonize` prioritizes deserialization to unsigned ints over signed ints, per https://github.com/davidhewitt/pythonize/pull/69.
3. I posed https://github.com/jamesbraza/tantivy-py/pull/1, which involves expanding `Document.__richcmp__` to recursively cast unsigned integers to signed integers.
4. In https://github.com/davidhewitt/pythonize/issues/80, David poses adding a config option for `depythonize` to prefer signed ints over unsigned ints.
5. Caleb just added a solution of, upon deserialization of `Object`, recursively casting numbers to I64 (if possible), otherwise F64.
    - This is more opinionated than https://github.com/jamesbraza/tantivy-py/pull/1 because it eliminates unsigned values in JSON. It's also more "`tantivy-py` native" by directly using `Object` (`SerdeValue` enum's item for JSON), instead of `Document.__richcmp__`.

In practice this change is backwards compatible ❤️ as far as I am concerned. The only incompatibility is round trip ser/de's an old `Document` will convert any unsigned ints to signed ints in JSON, but that doesn't matter. I am down to ship this 🚢 .

---

In other news, it looks like `pythonize::depythonize_bound` was just removed in `pythonize` version 0.24 in https://github.com/davidhewitt/pythonize/pull/83, so I just updated the code for that. This brings `cargo build` to pass.
author_association: CONTRIBUTOR