Repository: JuliaLang/julia
File: base/strings/unicode.jl (Julia)

Code before:
    ret < 0 && utf8proc_error(ret)
    return ret
end
function utf8proc_decompose(str, options, buffer, nwords, chartransform::F) where F
    ret = ccall(:utf8proc_decompose_custom, Int, (Ptr{UInt8}, Int, Ptr{UInt8}, Int, Cint, Ptr{Cvoid}, Ref{F}),
                str, sizeof(str), buffer, nwords, options,
                @cfunction(utf8proc_custom_func, UInt32, (UInt32, Ref{F})), chartransform)
    ret < 0 && utf8proc_error(ret)
    return ret
end

function utf8proc_map(str::Union{String,SubString{String}}, options::Integer, chartransform::F = identity) where F
    nwords = utf8proc_decompose(str, options, C_NULL, 0, chartransform)
    buffer = Base.StringVector(nwords*4)
    nwords = utf8proc_decompose(str, options, buffer, nwords, chartransform)
    nbytes = ccall(:utf8proc_reencode, Int, (Ptr{UInt8}, Int, Cint), buffer, nwords, options)
    nbytes < 0 && utf8proc_error(nbytes)
    return String(resize!(buffer, nbytes))
end

# Array of {original codepoint, replacement codepoint} normalizations
# to perform on Julia identifiers, to canonicalize characters that
# are both easily confused and easily inputted by accident.
#
# Important: when this table is updated, also update the corresponding table
#            in src/flisp/julia_charmap.h
const _julia_charmap = Dict{UInt32,UInt32}(
    0x025B => 0x03B5, # latin small letter open e -> greek small letter epsilon
    0x00B5 => 0x03BC, # micro sign -> greek small letter mu
    0x00B7 => 0x22C5, # middot char -> dot operator (#25098)
    0x0387 => 0x22C5, # Greek interpunct -> dot operator (#25098)
    0x2212 => 0x002D, # minus -> hyphen-minus (#26193)
    0x210F => 0x0127, # hbar -> small letter h with stroke (#48870)
)

utf8proc_map(s::AbstractString, flags::Integer, chartransform::F = identity) where F = utf8proc_map(String(s), flags, chartransform)

# Documented in Unicode module
function normalize(
    s::AbstractString;
    stable::Bool=false,
    compat::Bool=false,
    compose::Bool=true,
    decompose::Bool=false,
    stripignore::Bool=false,
    rejectna::Bool=false,
    newline2ls::Bool=false,
    newline2ps::Bool=false,
    newline2lf::Bool=false,
    stripcc::Bool=false,
    casefold::Bool=false,

Review comment (suggestion):
I guess it's nicer to make it a doc string, using Markdown:

```suggestion
"""
`Dict` of `original codepoint => replacement codepoint` normalizations
to perform on Julia identifiers, to canonicalize characters that
are both easily confused and easily inputted by accident.

!!! warning
    When this table is updated, also update the corresponding table
    in `src/flisp/julia_charmap.h`.
"""
```

Code after:
    ret < 0 && utf8proc_error(ret)
    return ret
end
function utf8proc_decompose(str, options, buffer, nwords, chartransform::F) where F
    ret = ccall(:utf8proc_decompose_custom, Int, (Ptr{UInt8}, Int, Ptr{UInt8}, Int, Cint, Ptr{Cvoid}, Ref{F}),
                str, sizeof(str), buffer, nwords, options,
                @cfunction(utf8proc_custom_func, UInt32, (UInt32, Ref{F})), chartransform)
    ret < 0 && utf8proc_error(ret)
    return ret
end

function utf8proc_map(str::Union{String,SubString{String}}, options::Integer, chartransform::F = identity) where F
    nwords = utf8proc_decompose(str, options, C_NULL, 0, chartransform)
    buffer = Base.StringVector(nwords*4)
    nwords = utf8proc_decompose(str, options, buffer, nwords, chartransform)
    nbytes = ccall(:utf8proc_reencode, Int, (Ptr{UInt8}, Int, Cint), buffer, nwords, options)
    nbytes < 0 && utf8proc_error(nbytes)
    return String(resize!(buffer, nbytes))
end

"""
`Dict` of `original codepoint => replacement codepoint` normalizations
to perform on Julia identifiers, to canonicalize characters that
are both easily confused and easily inputted by accident.

!!! warning
    When this table is updated, also update the corresponding table in `src/flisp/julia_charmap.h`.
"""
const _julia_charmap = Dict{UInt32,UInt32}(
    0x025B => 0x03B5, # latin small letter open e -> greek small letter epsilon
    0x00B5 => 0x03BC, # micro sign -> greek small letter mu
    0x00B7 => 0x22C5, # middot char -> dot operator (#25098)
    0x0387 => 0x22C5, # Greek interpunct -> dot operator (#25098)
    0x2212 => 0x002D, # minus -> hyphen-minus (#26193)
    0x210F => 0x0127, # hbar -> small letter h with stroke (#48870)
)

utf8proc_map(s::AbstractString, flags::Integer, chartransform::F = identity) where F = utf8proc_map(String(s), flags, chartransform)

# Documented in Unicode module
function normalize(
    s::AbstractString;
    stable::Bool=false,
    compat::Bool=false,
    compose::Bool=true,
    decompose::Bool=false,
    stripignore::Bool=false,
    rejectna::Bool=false,
    newline2ls::Bool=false,
    newline2ps::Bool=false,
    newline2lf::Bool=false,