Repository: LadybirdBrowser/ladybird
Pull request #4273: LibUnicode: avoid rejecting end-of-text position as a valid boundary
Author: login: mikiubo
Created: 2025-04-07T21:15:40Z | Merged: 2025-04-11T19:30:17Z

When the cursor was positioned at the end of text, attempting to move it left (using the left arrow key) would fail because align_boundary() was rejecting the end-of-text position as a valid boundary.


Diff:
diff --git a/Libraries/LibUnicode/Segmenter.cpp b/Libraries/LibUnicode/Segmenter.cpp
index db66f9e551610..63023995d887b 100644
--- a/Libraries/LibUnicode/Segmenter.cpp
+++ b/Libraries/LibUnicode/Segmenter.cpp
@@ -87,15 +87,13 @@ class SegmenterImpl : public Segmenter {
     virtual Optional<size_t> previous_boundary(size_t boundary, Inclusive inclusive) override
     {
         auto icu_boundary = align_boundary(boundary);
-        if (!icu_boundary.has_value())
-            return {};
 
         if (inclusive == Inclusive::Yes) {
-            if (static_cast<bool>(m_segmenter->isBoundary(*icu_boundary)))
-                return static_cast<size_t>(*icu_boundary);
+            if (static_cast<bool>(m_segmenter->isBoundary(icu_boundary)))
+                return static_cast<size_t>(icu_boundary);
         }
 
-        if (auto index = m_segmenter->preceding(*icu_boundary); index != icu::BreakIterator::DONE)
+        if (auto index = m_segmenter->preceding(icu_boundary); index != icu::BreakIterator::DONE)
             return static_cast<size_t>(index);
 
         return {};
@@ -104,15 +102,13 @@ class SegmenterImpl : public Segmenter {
     virtual Optional<size_t> next_boundary(size_t boundary, Inclusive inclusive) override
     {
         auto icu_boundary = align_boundary(boundary);
-        if (!icu_boundary.has_value())
-            return {};
 
         if (inclusive == Inclusive::Yes) {
-            if (static_cast<bool>(m_segmenter->isBoundary(*icu_boundary)))
-                return static_cast<size_t>(*icu_boundary);
+            if (static_cast<bool>(m_segmenter->isBoundary(icu_boundary)))
+                return static_cast<size_t>(icu_boundary);
         }
 
-        if (auto index = m_segmenter->following(*icu_boundary); index != icu::BreakIterator::DONE)
+        if (auto index = m_segmenter->following(icu_boundary); index != icu::BreakIterator::DONE)
             return static_cast<size_t>(index);
 
         return {};
@@ -177,25 +173,25 @@ class SegmenterImpl : public Segmenter {
     }
 
 private:
-    Optional<i32> align_boundary(size_t boundary)
+    i32 align_boundary(size_t boundary)
     {
         auto icu_boundary = static_cast<i32>(boundary);
 
         return m_segmented_text.visit(
-            [&](String const& text) -> Optional<i32> {
+            [&](String const& text) {
                 if (boundary >= text.byte_count())
-                    return {};
+                    return static_cast<i32>(text.byte_count());
 
                 U8_SET_CP_START(text.bytes().data(), 0, icu_boundary);
                 return icu_boundary;
             },
-            [&](icu::UnicodeString const& text) -> Optional<i32> {
+            [&](icu::UnicodeString const& text) {
                 if (icu_boundary >= text.length())
-                    return {};
+                    return text.length();
 
                 return text.getChar32Start(icu_boundary);
             },
-            [](Empty) -> Optional<i32> { VERIFY_NOT_REACHED(); });
+            [](Empty) -> i32 { VERIFY_NOT_REACHED(); });
     }
 
     void for_each_boundary(SegmentationCallback callback)
diff --git a/Tests/LibUnicode/TestSegmenter.cpp b/Tests/LibUnicode/TestSegmenter.cpp
index d40ea82cebf6b..13368ab84513d 100644
--- a/Tests/LibUnicode/TestSegmenter.cpp
+++ b/Tests/LibUnicode/TestSegmenter.cpp
@@ -136,11 +136,23 @@ TEST_CASE(out_of_bounds)
         auto segmenter = Unicode::Segmenter::create(Unicode::SegmenterGranularity::Word);
         segmenter->set_segmented_text(text);
 
-        auto result = segmenter->previous_boundary(text.byte_count());
+        auto result = segmenter->previous_boundary(text.byte_count() + 1);
+        EXPECT(result.has_value());
+
+        result = segmenter->next_boundary(text.byte_count() + 1);
         EXPECT(!result.has_value());
 
+        result = segmenter->previous_boundary(text.byte_count());
+        EXPECT(result.has_value());
+
         result = segmenter->next_boundary(text.byte_count());
         EXPECT(!result.has_value());
+
+        result = segmenter->next_boundary(0);
+        EXPECT(result.has_value());
+
+        result = segmenter->previous_boundary(0);
+        EXPECT(!result.has_value());
     }
     {
         auto text = MUST(AK::utf8_to_utf16("foo"sv));
@@ -148,10 +160,22 @@ TEST_CASE(out_of_bounds)
         auto segmenter = Unicode::Segmenter::create(Unicode::SegmenterGranularity::Word);
         segmenter->set_segmented_text(Utf16View { text });
 
-        auto result = segmenter->previous_boundary(text.size());
+        auto result = segmenter->previous_boundary(text.size() + 1);
+        EXPECT(result.has_value());
+
+        result = segmenter->next_boundary(text.size() + 1);
         EXPECT(!result.has_value());
 
+        result = segmenter->previous_boundary(text.size());
+        EXPECT(result.has_value());
+
         result = segmenter->next_boundary(text.size());
         EXPECT(!result.has_value());
+
+        result = segmenter->next_boundary(0);
+        EXPECT(result.has_value());
+
+        result = segmenter->previous_boundary(0);
+        EXPECT(!result.has_value());
     }
 }


Reviews:
user: login: trflynn89
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-08T11:47:07Z
user: login: mikiubo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T08:21:22Z
user: login: trflynn89
body: LGTM. I pushed a change just to remove a newline that didn't need to be added, but will merge.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T18:46:45Z

Review comments:
diff_hunk: @@ -183,14 +183,16 @@ class SegmenterImpl : public Segmenter {
 
         return m_segmented_text.visit(
             [&](String const& text) -> Optional<i32> {
-                if (boundary >= text.byte_count())
+                if (boundary > text.byte_count())
                     return {};
-
+                if (boundary == text.byte_count()) {
+                    return static_cast<i32>(boundary);
+                }
                 U8_SET_CP_START(text.bytes().data(), 0, icu_boundary);
                 return icu_boundary;
             },
             [&](icu::UnicodeString const& text) -> Optional<i32> {
-                if (icu_boundary >= text.length())
+                if (icu_boundary > text.length())
path: Libraries/LibUnicode/Segmenter.cpp
user: login: trflynn89
body: Couple of things here:

This now makes UTF-8 and UTF-16 behave differently. If we pass in `text.length()`, the UTF-16 impl will return 0:
https://github.com/unicode-org/icu/blob/89fe16ef3b77a6b98f7664f52338c8ff80e28096/icu4c/source/common/unistr.cpp#L883-L892

Whereas the UTF-8 impl will return the text length.

What if instead of returning an empty optional ever, we just cap the boundary to the text length in both cases? That would make this method become something like:
```c++
i32 align_boundary(size_t boundary)
{
    auto icu_boundary = static_cast<i32>(boundary);

    return m_segmented_text.visit(
        [&](String const& text) {
            if (boundary >= text.byte_count())
                return static_cast<i32>(text.byte_count());

            U8_SET_CP_START(text.bytes().data(), 0, icu_boundary);
            return icu_boundary;
        },
        [&](icu::UnicodeString const& text) {
            if (icu_boundary >= text.length())
                return text.length();

            return text.getChar32Start(icu_boundary);
        },
        [](Empty) -> i32 { VERIFY_NOT_REACHED(); });
}
```

(and callers would have to be updated to handle this no longer returning `Optional`)
created_at: 2025-04-08T11:44:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -136,10 +136,10 @@ TEST_CASE(out_of_bounds)
         auto segmenter = Unicode::Segmenter::create(Unicode::SegmenterGranularity::Word);
         segmenter->set_segmented_text(text);
 
-        auto result = segmenter->previous_boundary(text.byte_count());
+        auto result = segmenter->previous_boundary(text.byte_count() + 1);
path: Tests/LibUnicode/TestSegmenter.cpp
user: login: trflynn89
body: Let's not remove testing for what happens when we pass in `text.byte_count()`. Let's be sure to test passing in both `text.byte_count()` and `text.byte_count() + 1`.
created_at: 2025-04-08T11:45:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -183,14 +183,16 @@ class SegmenterImpl : public Segmenter {
 
         return m_segmented_text.visit(
             [&](String const& text) -> Optional<i32> {
-                if (boundary >= text.byte_count())
+                if (boundary > text.byte_count())
                     return {};
-
+                if (boundary == text.byte_count()) {
+                    return static_cast<i32>(boundary);
+                }
                 U8_SET_CP_START(text.bytes().data(), 0, icu_boundary);
                 return icu_boundary;
             },
             [&](icu::UnicodeString const& text) -> Optional<i32> {
-                if (icu_boundary >= text.length())
+                if (icu_boundary > text.length())
path: Libraries/LibUnicode/Segmenter.cpp
user: login: mikiubo
body: Ok, I made a fix, but I’m not sure about the behavior of this:
```
        auto result = segmenter->previous_boundary(text.byte_count() + 1);
        EXPECT(result.has_value());
```

I’m not sure if I need to check whether it’s truly out of bounds and not just capped?
@trflynn89 
created_at: 2025-04-09T08:21:22Z
author_association: CONTRIBUTOR

PR comments:
user: login: ladybird-bot
created_at: 2025-04-07T21:15:55Z
body: Hello!

One or more of the commit messages in this PR do not match the Ladybird [code submission policy](https://github.com/LadybirdBrowser/ladybird/blob/master/CONTRIBUTING.md#code-submission-policy), please check the `lint_commits` CI job for more details on which commits were flagged and why.
Please do not close this PR and open another, instead modify your commit message(s) with [git commit --amend](https://docs.github.com/en/pull-requests/committing-changes-to-your-project/creating-and-editing-commits/changing-a-commit-message) and force push those changes to update this PR.
author_association: COLLABORATOR
user: login: trflynn89
created_at: 2025-04-07T21:17:48Z
body: Please see https://github.com/LadybirdBrowser/ladybird/pull/3592#discussion_r1957230616. This will cause undefined behavior.
author_association: CONTRIBUTOR
user: login: mikiubo
created_at: 2025-04-07T21:25:09Z
body: > Please see [#3592 (comment)](https://github.com/LadybirdBrowser/ladybird/pull/3592#discussion_r1957230616). This will cause undefined behavior.

@trflynn89  ok i think i understand.. do you think this will be a better solution? 
https://github.com/LadybirdBrowser/ladybird/pull/3536

I also add a check before the call U8_SET_CP_START
let me know your opinion what is the best approach?
author_association: CONTRIBUTOR