Repository: JuliaLang/julia
File: base/char.jl (Julia)

Code before:
end
@noinline throw_invalid_char(c::AbstractChar) = throw(InvalidCharError(c))
@noinline throw_code_point_err(u::Integer) = throw(CodePointError(u))

function ismalformed(c::Char)
    u = bitcast(UInt32, c)
    l1 = leading_ones(u) << 3
    t0 = trailing_zeros(u) & 56
    (l1 == 8) | (l1 + t0 > 32) |
    (((u & 0x00c0c0c0) ⊻ 0x00808080) >> t0 != 0)
end

@inline is_overlong_enc(u::UInt32) = (u >> 24 == 0xc0) | (u >> 24 == 0xc1) | (u >> 21 == 0x0704) | (u >> 20 == 0x0f08)

function isoverlong(c::Char)
    u = bitcast(UInt32, c)
    is_overlong_enc(u)
end

# fallback: other AbstractChar types, by default, are assumed
#           not to support malformed or overlong encodings.

"""
    Base.ismalformed(c::AbstractChar) -> Bool

Return `true` if `c` represents malformed (non-Unicode) data according to the
encoding used by `c`. Defaults to `false` for non-`Char` types.

Any *non*-malformed `c` can be mapped to an integer codepoint
by [`codepoint(c)`](@ref); this includes codepoints that are
not valid Unicode characters ([`isvalid(c)`](@ref) is `false`).
For example, well-formed characters can include invalid Unicode
codepoints like `'\\U110000'`, unpaired surrogates such as `'\\ud800'`,
and can also include overlong encodings ([`Base.isoverlong`](@ref)).
Malformed data, in contrast, cannot be decoded to a codepoint
(`codepoint` will throw an exception).

See also [`Base.show_invalid`](@ref).
"""
ismalformed(c::AbstractChar) = false

"""
    Base.isoverlong(c::AbstractChar) -> Bool

Return `true` if `c` represents an overlong UTF-8 sequence. Defaults
to `false` for non-`Char` types.

See also [`Base.show_invalid`](@ref).
"""
isoverlong(c::AbstractChar) = false


Review comment (suggestion):
I would interpret "non-Unicode" to mean "not valid according to the Unicode spec" which IIUC is what `Base.isvalid` checks.

Code after:
struct CodePointError{T<:Integer} <: Exception
    code::T
end
@noinline throw_invalid_char(c::AbstractChar) = throw(InvalidCharError(c))
@noinline throw_code_point_err(u::Integer) = throw(CodePointError(u))

function ismalformed(c::Char)
    u = bitcast(UInt32, c)
    l1 = leading_ones(u) << 3
    t0 = trailing_zeros(u) & 56
    (l1 == 8) | (l1 + t0 > 32) |
    (((u & 0x00c0c0c0) ⊻ 0x00808080) >> t0 != 0)
end

@inline is_overlong_enc(u::UInt32) = (u >> 24 == 0xc0) | (u >> 24 == 0xc1) | (u >> 21 == 0x0704) | (u >> 20 == 0x0f08)

function isoverlong(c::Char)
    u = bitcast(UInt32, c)
    is_overlong_enc(u)
end

# fallback: other AbstractChar types, by default, are assumed
#           not to support malformed or overlong encodings.

"""
    ismalformed(c::AbstractChar)::Bool

Return `true` if `c` represents malformed (non-codepoint / mis-encoded) data according to the
encoding used by `c`. Defaults to `false` for non-`Char` types.

Any *non*-malformed `c` can be mapped to an integer codepoint
by [`codepoint(c)`](@ref); this includes codepoints that are
not valid Unicode characters ([`isvalid(c)`](@ref) is `false`).
For example, well-formed characters can include invalid Unicode
codepoints like `'\\U110000'`, unpaired surrogates such as `'\\ud800'`,
and can also include overlong encodings ([`Base.isoverlong`](@ref)).
Malformed data, in contrast, cannot be decoded to a codepoint
(`codepoint` will throw an exception).

See also [`Base.show_invalid`](@ref).
"""
ismalformed(c::AbstractChar) = false

"""
    isoverlong(c::AbstractChar)::Bool

Return `true` if `c` represents an overlong UTF-8 sequence. Defaults
to `false` for non-`Char` types.

See also [`Base.show_invalid`](@ref).
"""