Repository: microsoft/terminal
File: src/inc/til/u8u16convert.h (C/C++)

Code before:
            if (state.have)
            {
                if (state.want > len8) // we still didn't get enough data to complete the code point, however this is not an error
                {
                    std::move(cursor8, cursor8 + len8, state.partials + state.have);
                    state.have += gsl::narrow_cast<uint8_t>(len8);
                    state.want -= gsl::narrow_cast<uint8_t>(len8);
                    out.clear();
                    return S_OK;
                }

                std::move(cursor8, cursor8 + state.want, state.partials + state.have);
                len16 = MultiByteToWideChar(CP_UTF8, 0UL, state.partials, gsl::narrow_cast<int>(state.have) + state.want, out.data(), capa16);
                RETURN_HR_IF(E_UNEXPECTED, !len16);

                capa16 -= len16;
                len8 -= state.want;
                cursor8 += state.want;
                state.reset();
            }

            if (len8)
            {
                auto backIter{ cursor8 + len8 };
                // If the last byte in the string was a byte belonging to a UTF-8 multi-byte character
                if (backIter[-1] & 0b1'0000000)
                {
                    // Check only up to 3 last bytes, if no Lead Byte is found then the byte before will be the Lead Byte and no partials are in the string
                    const auto stopLen{ std::min(len8, 3) };
                    for (auto sequenceLen{ 1 }; sequenceLen <= stopLen; ++sequenceLen)
                    {
                        --backIter;
                        // we found a Lead Byte if at least 2 high-order bits are set
                        if ((*backIter & 0b11'000000) == 0b11'000000)
                        {
                            // determine the number of bytes of the code point indicated by the lead byte
                            const auto codePointLen{
                                2 + // the code point consists of at least 2 bytes because we found a lead byte
                                gsl::narrow_cast<int>((*backIter & 0b111'00000) == 0b111'00000) + // add one if at least 3 high-order bits of the lead byte are set
                                gsl::narrow_cast<int>((*backIter & 0b1111'0000) == 0b1111'0000) // add another one if 4 high-order bits of the lead byte are set
                            };

                            // if the length of the code point differs from the length we stepped backward, we will capture the last sequence which is a partial code point
                            if (codePointLen != sequenceLen)
                            {
                                std::move(backIter, backIter + sequenceLen, state.partials);
                                len8 -= sequenceLen;
                                state.have = gsl::narrow_cast<uint8_t>(sequenceLen);
                                state.want = gsl::narrow_cast<uint8_t>(codePointLen - sequenceLen);
                            }


Review comment (bug):
BTW Tangential to the other comment-thread we have: Would it make sense to remove this if condition and only handle case where the trailing sequence is a valid partial sequence?
That way we could remove some of the if conditions all together... Something roughly like:
```cpp
auto backIter = cursor8 + len8 - 1;
int sequenceLen = 1;

// Skip UTF8 continuation bytes
while (backIter != cursor8 && (*backIter & 0b11'000000) == 0b10'000000)
{
    ++sequenceLen;
}

if ((*backIter & 0b11'000000) == 0b11'000000)
{
    static constexpr uint8_t lengths[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 3, 3, 4, 0};
    const auto codePointLen = lengths[ch >> 3];

    if (codePointLen > sequenceLen)
    {
        std::move(backIter, backIter + sequenceLen, state.partials);
        len8 -= sequenceLen;
        state.have = gsl::narrow_cast<uint8_t>(sequenceLen);
        state.want = gsl::narrow_cast<uint8_t>(codePointLen - sequenceLen);
    }
}
```

Unless I did a off-by-one error or something similar this should properly work for all given inputs but is a bit less code and more linear.
It doesn't have to stop early after 3 bytes IMO as performance doesn't necessarily need to be guaranteed for vastly invalid inputs. This code should still be sufficiently fast (~3GB/s for backwards byte-wise iterations like that).

Code after:
                }

                len16 = MultiByteToWideChar(CP_UTF8, 0UL, &state.partials[0], gsl::narrow_cast<int>(state.have), out.data(), capa16);
                RETURN_HR_IF(E_UNEXPECTED, !len16);

                capa16 -= len16;
                len8 -= copyable;
                cursor8 += copyable;
                // state.want is already zero at this point
                state.have = 0;
            }

            if (len8)
            {
                auto backIter{ cursor8 + len8 - 1 };
                int sequenceLen{ 1 };

                // skip UTF8 continuation bytes
                while (backIter != cursor8 && (*backIter & 0b11'000000) == 0b10'000000)
                {
                    --backIter;
                    ++sequenceLen;
                }

                // credits go to Christopher Wellons for this algorithm to determine the length of a UTF-8 code point
                // it is released into the Public Domain. https://github.com/skeeto/branchless-utf8
                static constexpr uint8_t lengths[]{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 3, 3, 4, 0 };
                const auto codePointLen{ lengths[gsl::narrow_cast<uint8_t>(*backIter) >> 3] };

                if (codePointLen > sequenceLen)
                {
                    std::move(backIter, backIter + sequenceLen, &state.partials[0]);
                    len8 -= sequenceLen;
                    state.have = gsl::narrow_cast<uint8_t>(sequenceLen);
                    state.want = gsl::narrow_cast<uint8_t>(codePointLen - sequenceLen);
                }
            }

            if (len8)
            {
                const auto convLen{ MultiByteToWideChar(CP_UTF8, 0UL, cursor8, len8, out.data() + len16, capa16) };
                RETURN_HR_IF(E_UNEXPECTED, !convLen);

                len16 += convLen;
            }

            out.resize(gsl::narrow_cast<size_t>(len16));
            return S_OK;
        }
        CATCH_RETURN();
    }