Repository: ashvardanian/StringZilla (language: rust)

Issue:
Pretty-printing `Strs` in Python
The `Str` Python class already provides nice visualizations for Jupyter. Let's provide one for `Strs` and `File`.

![image](https://github.com/ashvardanian/StringZilla/assets/1983160/0b2ad990-cd16-4bf3-9ee2-1ec3bc6b826d)



Discussion / hints:
Do we want the `File` representation to be similar to the Python built-in file type? That is, display the user-provided/default parameters rather than its (truncated) contents?

Here is an example of how Python does it:

```
>>> open("sample_data/california_housing_test.csv", "r")
<_io.TextIOWrapper name='sample_data/california_housing_test.csv' mode='r' encoding='UTF-8'>
```

Also, how is this for a representation of `Strs`: `sz.Strs([sz.Str('A'), sz.Str('B'), ..., sz.Str('Z')])`

(I _think_ Jupyter handles the truncation in its own formatter so the `Strs_repr` function might not need to consider it, but I need to look into that)
@alexbowe, I think the `sz.Strs([sz.Str('A'), sz.Str('B'), ..., sz.Str('Z')])` may be too mouthful, but the `sz.Strs(['A', 'B', 'B'...])` would be great!

Fix patch:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index ffbf94bd..d4ddaff2 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -28,6 +28,11 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: 20
+      - name: Set up Cargo
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
       - run: npm install --ignore-scripts --save-dev --prefix ./package-ci @semantic-release/exec @semantic-release/git conventional-changelog-eslint semantic-release && npx --prefix ./package-ci semantic-release
 
   rebase:
diff --git a/.github/workflows/update_version.sh b/.github/workflows/update_version.sh
index a2fcaeec..fc70eda3 100644
--- a/.github/workflows/update_version.sh
+++ b/.github/workflows/update_version.sh
@@ -7,3 +7,7 @@ echo $1 > VERSION &&
     sed -i "s/^version = \".*\"/version = \"$1\"/" Cargo.toml &&
     sed -i "s/\"version\": \".*\"/\"version\": \"$1\"/" package.json &&
     sed -i "s/VERSION [0-9]\+\.[0-9]\+\.[0-9]\+/VERSION $1/" CMakeLists.txt
+
+# Update the version in the Cargo.lock file, but don't report an error if it fails...
+# as `cargo` may not be available in the current environment.
+cargo update || true
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 84898b1a..8510d3d9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -168,7 +168,7 @@ function(set_compiler_flags target cpp_standard target_arch)
   if("${target_arch}" STREQUAL "")
     # Only use the current system if we are not cross compiling
     if((NOT CMAKE_CROSSCOMPILING) OR (CMAKE_SYSTEM_PROCESSOR MATCHES CMAKE_HOST_SYSTEM_PROCESSOR))
-      if (TRUE)
+      if (NOT MSVC)
         include(CheckCXXCompilerFlag)
         check_cxx_compiler_flag("-march=native" supports_march_native)
         if (supports_march_native)
diff --git a/Cargo.lock b/Cargo.lock
index 524f8167..ba62a8b8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -19,7 +19,7 @@ checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"
 
 [[package]]
 name = "stringzilla"
-version = "3.7.2"
+version = "3.7.3"
 dependencies = [
  "cc",
 ]
diff --git a/include/stringzilla/stringzilla.h b/include/stringzilla/stringzilla.h
index aba40060..4d11ddb4 100644
--- a/include/stringzilla/stringzilla.h
+++ b/include/stringzilla/stringzilla.h
@@ -1190,7 +1190,7 @@ SZ_PUBLIC void sz_hashes_avx512(sz_cptr_t text, sz_size_t length, sz_size_t wind
 #endif
 
 #if SZ_USE_X86_AVX2
-/** @copydoc sz_equal */
+/** @copydoc sz_copy */
 SZ_PUBLIC void sz_copy_avx2(sz_ptr_t target, sz_cptr_t source, sz_size_t length);
 /** @copydoc sz_move */
 SZ_PUBLIC void sz_move_avx2(sz_ptr_t target, sz_cptr_t source, sz_size_t length);
diff --git a/python/lib.c b/python/lib.c
index a178951f..43b06d29 100644
--- a/python/lib.c
+++ b/python/lib.c
@@ -438,7 +438,7 @@ static PyObject *File_new(PyTypeObject *type, PyObject *positional_args, PyObjec
 }
 
 static int File_init(File *self, PyObject *positional_args, PyObject *named_args) {
-    const char *path;
+    char const *path;
     if (!PyArg_ParseTuple(positional_args, "s", &path)) return -1;
 
 #if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
@@ -837,6 +837,7 @@ static PyObject *Strs_subscript(Strs *self, PyObject *key) {
 
     // Create a new `Strs` object
     Strs *result = (Strs *)StrsType.tp_alloc(&StrsType, 0);
+    // REVIEW(alexbowe): Does this raise the appropriate Error on the Python side?
     if (result == NULL && PyErr_NoMemory()) return NULL;
     if (result_count == 0) {
         result->type = STRS_REORDERED;
@@ -904,7 +905,7 @@ static PyObject *Strs_subscript(Strs *self, PyObject *key) {
         size_t element_length;
         str_at_offset_consecutive_32bit(self, start, count, &to->parent_string, &to->start, &element_length);
         to->end_offsets[0] = element_length;
-        for (size_t i = 1; i < result_count; ++i) {
+        for (Py_ssize_t i = 1; i < result_count; ++i) {
             to->end_offsets[i - 1] += from->separator_length;
             PyObject *element_parent = NULL;
             char const *element_start = NULL;
@@ -933,7 +934,7 @@ static PyObject *Strs_subscript(Strs *self, PyObject *key) {
         size_t element_length;
         str_at_offset_consecutive_64bit(self, start, count, &to->parent_string, &to->start, &element_length);
         to->end_offsets[0] = element_length;
-        for (size_t i = 1; i < result_count; ++i) {
+        for (Py_ssize_t i = 1; i < result_count; ++i) {
             to->end_offsets[i - 1] += from->separator_length;
             PyObject *element_parent = NULL;
             char const *element_start = NULL;
@@ -1288,7 +1289,7 @@ static PyObject *Str_write_to(PyObject *self, PyObject *args, PyObject *kwargs)
     setbuf(file_pointer, NULL); // Set the stream to unbuffered
     int status = fwrite(text.start, 1, text.length, file_pointer);
     PyEval_RestoreThread(gil_state);
-    if (status != text.length) {
+    if (status != (Py_ssize_t)text.length) {
         PyErr_SetFromErrnoWithFilename(PyExc_OSError, path_buffer);
         free(path_buffer);
         fclose(file_pointer);
@@ -2907,18 +2908,165 @@ static PyObject *Strs_sample(Strs *self, PyObject *args, PyObject *kwargs) {
 }
 
 /**
- *  @brief  Array to string conversion method, that concatenates all the strings in the array.
+ *  @brief Exports a string to a UTF-8 buffer, escaping single quotes.
+ *  @param[out] did_fit Populated with 1 if the string is fully exported, 0 if it didn't fit.
  */
-static PyObject *Strs_str(Strs *self) {
-    // This is just an example, adapt it to your needs
-    // For instance, you could iterate over your Strs and concatenate them into a single string
-    return PyUnicode_FromFormat("<%s object at %p>", Py_TYPE(self)->tp_name, self);
+char const *export_escaped_unquoted_to_utf8_buffer(char const *cstr, size_t cstr_length, //
+                                                   char *buffer, size_t buffer_length,   //
+                                                   int *did_fit) {
+    char const *const cstr_end = cstr + cstr_length;
+    char *const buffer_end = buffer + buffer_length;
+    *did_fit = 1;
+
+    while (cstr < cstr_end) {
+        sz_rune_t rune;
+        sz_rune_length_t rune_length;
+        _sz_extract_utf8_rune(cstr, &rune, &rune_length);
+        if (rune_length == 1 && buffer + 2 < buffer_end) {
+            if (*cstr == '\'') {
+                *(buffer++) = '\\';
+                *(buffer++) = '\'';
+                cstr++;
+            }
+            else if (*cstr == '\'') {
+                *(buffer++) = '\\';
+                *(buffer++) = '\'';
+                cstr++;
+            }
+            else { *(buffer++) = *(cstr++); }
+        }
+        else if (buffer + rune_length < buffer_end) {
+            memcpy(buffer, cstr, rune_length);
+            buffer += rune_length;
+            cstr += rune_length;
+        }
+        else {
+            *did_fit = 0;
+            break;
+        }
+    }
+
+    return buffer;
 }
 
+/**
+ *  @brief  Formats an array of strings, similar to the `repr` method of Python lists.
+ *          Will output an object that looks like `sz.Str(['item1', 'item2... ])`, potentially
+ *          dropping the last few entries.
+ */
 static PyObject *Strs_repr(Strs *self) {
-    // This is just an example, adapt it to your needs
-    // For instance, you could iterate over your Strs and concatenate them into a single string
-    return PyUnicode_FromFormat("<%s object at %p>", Py_TYPE(self)->tp_name, self);
+    get_string_at_offset_t getter = str_at_offset_getter(self);
+    if (!getter) {
+        PyErr_SetString(PyExc_TypeError, "Unknown Strs kind");
+        return NULL;
+    }
+
+    char repr_buffer[1024];
+    char *repr_buffer_ptr = &repr_buffer[0];
+    char const *const repr_buffer_end = repr_buffer_ptr + 1024;
+
+    // Start of the array
+    memcpy(repr_buffer_ptr, "sz.Strs([", 9);
+    repr_buffer_ptr += 9;
+
+    size_t count = Strs_len(self);
+    PyObject *parent_string;
+
+    // In the worst case, we must have enough space for `...', ...])`
+    // That's extra 11 bytes of content.
+    char const *non_fitting_array_tail = "... ])";
+    int const non_fitting_array_tail_length = 6;
+
+    // If the whole string doesn't fit, even before the `non_fitting_array_tail` tail,
+    // we need to add `, '` separator of 3 bytes.
+    for (size_t i = 0; i < count && repr_buffer_ptr + (non_fitting_array_tail_length + 3) < repr_buffer_end; i++) {
+        char const *cstr_start = NULL;
+        size_t cstr_length = 0;
+        getter(self, i, count, &parent_string, &cstr_start, &cstr_length);
+
+        if (i > 0) { *(repr_buffer_ptr++) = ',', *(repr_buffer_ptr++) = ' '; }
+        *(repr_buffer_ptr++) = '\'';
+
+        int did_fit;
+        repr_buffer_ptr = export_escaped_unquoted_to_utf8_buffer(
+            cstr_start, cstr_length, repr_buffer_ptr, repr_buffer_end - repr_buffer_ptr - non_fitting_array_tail_length,
+            &did_fit);
+        // If it didn't fit, let's put an ellipsis
+        if (!did_fit) {
+            memcpy(repr_buffer_ptr, non_fitting_array_tail, non_fitting_array_tail_length);
+            repr_buffer_ptr += non_fitting_array_tail_length;
+            return PyUnicode_FromStringAndSize(repr_buffer, repr_buffer_ptr - repr_buffer);
+        }
+        else
+            *(repr_buffer_ptr++) = '\''; // Close the string
+    }
+
+    // Close the array
+    *(repr_buffer_ptr++) = ']', *(repr_buffer_ptr++) = ')';
+    return PyUnicode_FromStringAndSize(repr_buffer, repr_buffer_ptr - repr_buffer);
+}
+
+/**
+ *  @brief  Array to string conversion method, that concatenates all the strings in the array.
+ *          Will output an object that looks like `['item1', 'item2', 'item3']`, containing all
+ *          the strings.
+ */
+static PyObject *Strs_str(Strs *self) {
+    get_string_at_offset_t getter = str_at_offset_getter(self);
+    if (!getter) {
+        PyErr_SetString(PyExc_TypeError, "Unknown Strs kind");
+        return NULL;
+    }
+
+    // Aggregate the total length of all the slices and count the number of bytes we need to allocate:
+    size_t count = Strs_len(self);
+    PyObject *parent_string;
+    size_t total_bytes = 2; // opening and closing square brackets
+    for (size_t i = 0; i < count; i++) {
+        char const *cstr_start = NULL;
+        size_t cstr_length = 0;
+        getter(self, i, count, &parent_string, &cstr_start, &cstr_length);
+        total_bytes += cstr_length;
+        total_bytes += 2;             // For the single quotes
+        if (i != 0) total_bytes += 2; // For the preceding comma and space
+
+        // Count the number of single quotes in the string
+        while (cstr_length) {
+            char quote = '\'';
+            sz_cptr_t next_quote = sz_find_byte(cstr_start, cstr_length, &quote);
+            if (next_quote == NULL) break;
+            total_bytes++;
+            cstr_length -= next_quote - cstr_start;
+            cstr_start = next_quote + 1;
+        }
+    }
+
+    // Now allocate the memory for the concatenated string
+    char *const result_buffer = malloc(total_bytes);
+    if (!result_buffer) {
+        PyErr_SetString(PyExc_MemoryError, "Failed to allocate memory for the concatenated string");
+        return NULL;
+    }
+
+    // Copy the strings into the result buffer
+    char *result_ptr = result_buffer;
+    *result_ptr++ = '[';
+    for (size_t i = 0; i < count; i++) {
+        if (i != 0) {
+            *result_ptr++ = ',';
+            *result_ptr++ = ' ';
+        }
+        char const *cstr_start = NULL;
+        size_t cstr_length = 0;
+        getter(self, i, count, &parent_string, &cstr_start, &cstr_length);
+        *result_ptr++ = '\'';
+        int did_fit;
+        result_ptr = export_escaped_unquoted_to_utf8_buffer(cstr_start, cstr_length, result_ptr, total_bytes, &did_fit);
+        *result_ptr++ = '\'';
+    }
+
+    *result_ptr++ = ']';
+    return PyUnicode_FromStringAndSize(result_buffer, total_bytes);
 }
 
 static PySequenceMethods Strs_as_sequence = {
@@ -2949,7 +3097,8 @@ static PyMethodDef Strs_methods[] = {
     {"sort", Strs_sort, SZ_METHOD_FLAGS, "Sort (in-place) the elements of the Strs object."},          //
     {"order", Strs_order, SZ_METHOD_FLAGS, "Provides the indexes to achieve sorted order."},           //
     {"sample", Strs_sample, SZ_METHOD_FLAGS, "Provides a random sample of a given size."},             //
-    // {"to_pylist", Strs_to_pylist, SZ_METHOD_FLAGS, "Exports string-views to a native list of native strings."}, //
+    // {"to_pylist", Strs_to_pylist, SZ_METHOD_FLAGS, "Exports string-views to a native list of native strings."},
+    // //
     {NULL, NULL, 0, NULL}};
 
 static PyTypeObject StrsType = {
@@ -2999,11 +3148,13 @@ static PyMethodDef stringzilla_methods[] = {
     {"hamming_distance", Str_hamming_distance, SZ_METHOD_FLAGS,
      "Hamming distance between two strings, as the number of replaced bytes, and difference in length."},
     {"hamming_distance_unicode", Str_hamming_distance_unicode, SZ_METHOD_FLAGS,
-     "Hamming distance between two strings, as the number of replaced unicode characters, and difference in length."},
+     "Hamming distance between two strings, as the number of replaced unicode characters, and difference in "
+     "length."},
     {"edit_distance", Str_edit_distance, SZ_METHOD_FLAGS,
      "Levenshtein distance between two strings, as the number of inserted, deleted, and replaced bytes."},
     {"edit_distance_unicode", Str_edit_distance_unicode, SZ_METHOD_FLAGS,
-     "Levenshtein distance between two strings, as the number of inserted, deleted, and replaced unicode characters."},
+     "Levenshtein distance between two strings, as the number of inserted, deleted, and replaced unicode "
+     "characters."},
     {"alignment_score", Str_alignment_score, SZ_METHOD_FLAGS,
      "Needleman-Wunsch alignment score given a substitution cost matrix."},
 


Test patch:
diff --git a/scripts/test.py b/scripts/test.py
index cd8df29f..f4bf8628 100644
--- a/scripts/test.py
+++ b/scripts/test.py
@@ -31,7 +31,7 @@
 
     pyarrow_available = True
 except:
-    # NumPy is not installed, most tests will be skipped
+    # PyArrow is not installed, most tests will be skipped
     pyarrow_available = False
 
 
@@ -46,6 +46,13 @@ def test_unit_construct():
     assert len(big) == len(native)
 
 
+def test_str_repr():
+    native = "abcdef"
+    big = Str(native)
+    assert repr(big) == f"sz.Str({repr(native)})"
+    assert str(big) == native
+
+
 def test_unit_indexing():
     native = "abcdef"
     big = Str(native)
@@ -258,6 +265,13 @@ def test_unit_strs_sequence():
     assert "p3" in lines
     assert "p4" not in lines
 
+    assert repr(lines) == "sz.Strs(['p3', 'p2', 'p1'])"
+    assert repr(Str("a" * 1_000_000).split()).endswith("... ])")
+
+    assert str(lines) == "['p3', 'p2', 'p1']"
+    assert str(Str("a" * 1_000_000).split()).startswith("['aaa")
+    assert str(Str("a" * 1_000_000).split()).endswith("aaa']")
+
     lines.sort()
     assert [0, 1, 2] == list(lines.order())
     assert ["p1", "p2", "p3"] == list(lines)
@@ -729,3 +743,9 @@ def test_pyarrow_str_conversion():
 
     arrow_buffer = pa.foreign_buffer(big.address, big.nbytes, big)
     assert arrow_buffer.to_pybytes() == native.encode("utf-8")
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-x", "-s", __file__]))