D-Programming-Language/phobos — pull request #1477 by monarchdodra (2013-08-15 10:08:03+00:00)
Title: std.array.split Unicode correctness

This makes std.array.split unicode correct, by splitting over unicode whites (as opposed to ascii whites). The algorithm was improved to:
1. Use `std.uni.isWhite`
2. Use `std.array.Appender` which is now safe and CTFE-able
3. Rework the algorithm for better speed: The old algo was not very efficient for Unicode iteration.

FYI, this is the 4th attempt at this. This one is (I think) the simplest yet.

Algo is:

```D
S[] split(S)(S s) if (isSomeString!S)
{
//Start by stripping the white out of the right hand side.
//This allows faster iteration later: we know that is white is always followed by text
s = s.stripRight();
if (s.empty)
return S[].init;

auto result = appender!(S[])();

immutable len = s.length;
size_t startIndex; //word start
static if (isNarrowString!S)
{
size_t nextIndex; //index of the next letter

//doing this instead of stripLeft means not parsing the first character twice.
for ( ; ; startIndex = nextIndex )
if(!std.uni.isWhite(decode(s, nextIndex)))
break;

find_token:
for ( size_t currIndex = nextIndex ; currIndex < len ; currIndex = nextIndex )
{
if (std.uni.isWhite(decode(s, nextIndex)))
{
//White found. Append word.
result.put(s[startIndex .. currIndex]);

//strip remaining white and prepare next iteration.
for ( startIndex = nextIndex ; ; startIndex = nextIndex )
if(!std.uni.isWhite(decode(s, nextIndex)))
goto find_token;
}
}
}
else
{
//doing this instead of stripLeft means not parsing the very first character twice.
for ( ; ; ++startIndex )
if (!std.uni.isWhite(s[startIndex]))
break;

find_token:
for ( size_t currIndex = startIndex + 1 ; currIndex < len ; ++currIndex )
{
if (std.uni.isWhite(s[currIndex]))
{
//White found. Append word.
result.put(s[startIndex .. currIndex]);

//strip remaining white and prepare next iteration.
for ( startIndex = currIndex + 1 ; ; ++startIndex )
if (!std.uni.isWhite(s[startIndex]))
goto find_token;
}
}
}
//The last token
result.put(s[startIndex .. len]);

return result.data;
}
```