Commit message: deps: fix out-of-band write in utf8 decoder

Originally reported by: Kris Reeves <kris.re@bbhmedia.com>

Reviewed-By: Trevor Norris <trev.norris@gmail.com>
CVE: CVE-2015-5380
CWE: CWE-119
Vulnerability description: The Utf8DecoderBase::WriteUtf16Slow function in unicode-decoder.cc in Google V8, as used in Node.js before 0.12.6, io.js before 1.8.3 and 2.x before 2.3.3, and other products, does not verify that there is memory available for a UTF-16 surrogate pair, which allows remote attackers to cause a denial of service (memory corruption) or possibly have unspecified other impact via a crafted byte sequence.

Project: node
File: unicode.cc

static inline bool IsStart(int32_t entry) {
  return (entry & kStartBit) != 0;
}

Project: node
File: unicode-inl.h

Utf8DecoderBase::Utf8DecoderBase()
  : unbuffered_start_(NULL),
    unbuffered_length_(0),
    utf16_length_(0),
    last_byte_of_buffer_unused_(false) {}

Project: node
File: unicode.cc

static int LookupMapping(const int32_t* table,
                         uint16_t size,
                         const MultiCharacterSpecialCase<kW>* multi_chars,
                         uchar chr,
                         uchar next,
                         uchar* result,
                         bool* allow_caching_ptr) {
  static const int kEntryDist = 2;
  uint16_t key = chr & (kChunkBits - 1);
  uint16_t chunk_start = chr - key;
  unsigned int low = 0;
  unsigned int high = size - 1;
  while (high != low) {
    unsigned int mid = low + ((high - low) >> 1);
    uchar current_value = GetEntry(TableGet<kEntryDist>(table, mid));
    // If we've found an entry less than or equal to this one, and the next one
    // is not also less than this one, we've arrived.
    if ((current_value <= key) &&
        (mid + 1 == size ||
         GetEntry(TableGet<kEntryDist>(table, mid + 1)) > key)) {
      low = mid;
      break;
    } else if (current_value < key) {
      low = mid + 1;
    } else if (current_value > key) {
      // If we've just checked the bottom-most value and it's not
      // the one we're looking for, we're done.
      if (mid == 0) break;
      high = mid - 1;
    }
  }
  int32_t field = TableGet<kEntryDist>(table, low);
  uchar entry = GetEntry(field);
  bool is_start = IsStart(field);
  bool found = (entry == key) || (entry < key && is_start);
  if (found) {
    int32_t value = table[2 * low + 1];
    if (value == 0) {
      // 0 means not present
      return 0;
    } else if ((value & 3) == 0) {
      // Low bits 0 means a constant offset from the given character.
      if (ranges_are_linear) {
        result[0] = chr + (value >> 2);
      } else {
        result[0] = entry + chunk_start + (value >> 2);
      }
      return 1;
    } else if ((value & 3) == 1) {
      // Low bits 1 means a special case mapping
      if (allow_caching_ptr) *allow_caching_ptr = false;
      const MultiCharacterSpecialCase<kW>& mapping = multi_chars[value >> 2];
      int length = 0;
      for (length = 0; length < kW; length++) {
        uchar mapped = mapping.chars[length];
        if (mapped == MultiCharacterSpecialCase<kW>::kEndOfEncoding) break;
        if (ranges_are_linear) {
          result[length] = mapped + (key - entry);
        } else {
          result[length] = mapped;
        }
      }
      return length;
    } else {
      // Low bits 2 means a really really special case
      if (allow_caching_ptr) *allow_caching_ptr = false;
      // The cases of this switch are defined in unicode.py in the
      // really_special_cases mapping.
      switch (value >> 2) {
        case 1:
          // Really special case 1: upper case sigma.  This letter
          // converts to two different lower case sigmas depending on
          // whether or not it occurs at the end of a word.
          if (next != 0 && Letter::Is(next)) {
            result[0] = 0x03C3;
          } else {
            result[0] = 0x03C2;
          }
          return 1;
        default:
          return 0;
      }
      return -1;
    }
  } else {
    return 0;
  }
}

Project: node
File: unicode-inl.h

Utf8Decoder<kBufferSize>::Utf8Decoder(const char* stream, unsigned length)
  : Utf8DecoderBase(buffer_,
                    kBufferSize,
                    reinterpret_cast<const uint8_t*>(stream),
                    length) {
}

Project: node
File: unicode.cc

int Ecma262Canonicalize::Convert(uchar c,
                      uchar n,
                      uchar* result,
                      bool* allow_caching_ptr) {
  int chunk_index = c >> 13;
  switch (chunk_index) {
    case 0: return LookupMapping<true>(kEcma262CanonicalizeTable0,
                                           kEcma262CanonicalizeTable0Size,
                                           kEcma262CanonicalizeMultiStrings0,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 1: return LookupMapping<true>(kEcma262CanonicalizeTable1,
                                           kEcma262CanonicalizeTable1Size,
                                           kEcma262CanonicalizeMultiStrings1,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 5: return LookupMapping<true>(kEcma262CanonicalizeTable5,
                                           kEcma262CanonicalizeTable5Size,
                                           kEcma262CanonicalizeMultiStrings5,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 7: return LookupMapping<true>(kEcma262CanonicalizeTable7,
                                           kEcma262CanonicalizeTable7Size,
                                           kEcma262CanonicalizeMultiStrings7,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    default: return 0;
  }
}

Project: node
File: unicode-inl.h

unsigned Utf8::Encode(char* str,
                      uchar c,
                      int previous,
                      bool replace_invalid) {
  static const int kMask = ~(1 << 6);
  if (c <= kMaxOneByteChar) {
    str[0] = c;
    return 1;
  } else if (c <= kMaxTwoByteChar) {
    str[0] = 0xC0 | (c >> 6);
    str[1] = 0x80 | (c & kMask);
    return 2;
  } else if (c <= kMaxThreeByteChar) {
    if (Utf16::IsSurrogatePair(previous, c)) {
      const int kUnmatchedSize = kSizeOfUnmatchedSurrogate;
      return Encode(str - kUnmatchedSize,
                    Utf16::CombineSurrogatePair(previous, c),
                    Utf16::kNoPreviousCharacter,
                    replace_invalid) - kUnmatchedSize;
    } else if (replace_invalid &&
               (Utf16::IsLeadSurrogate(c) ||
               Utf16::IsTrailSurrogate(c))) {
      c = kBadChar;
    }
    str[0] = 0xE0 | (c >> 12);
    str[1] = 0x80 | ((c >> 6) & kMask);
    str[2] = 0x80 | (c & kMask);
    return 3;
  } else {
    str[0] = 0xF0 | (c >> 18);
    str[1] = 0x80 | ((c >> 12) & kMask);
    str[2] = 0x80 | ((c >> 6) & kMask);
    str[3] = 0x80 | (c & kMask);
    return 4;
  }
}

Project: node
File: unicode.cc

int ToLowercase::Convert(uchar c,
                      uchar n,
                      uchar* result,
                      bool* allow_caching_ptr) {
  int chunk_index = c >> 13;
  switch (chunk_index) {
    case 0: return LookupMapping<true>(kToLowercaseTable0,
                                           kToLowercaseTable0Size,
                                           kToLowercaseMultiStrings0,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 1: return LookupMapping<true>(kToLowercaseTable1,
                                           kToLowercaseTable1Size,
                                           kToLowercaseMultiStrings1,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 5: return LookupMapping<true>(kToLowercaseTable5,
                                           kToLowercaseTable5Size,
                                           kToLowercaseMultiStrings5,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 7: return LookupMapping<true>(kToLowercaseTable7,
                                           kToLowercaseTable7Size,
                                           kToLowercaseMultiStrings7,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    default: return 0;
  }
}

Project: node
File: unicode-inl.h

void Utf8Decoder<kBufferSize>::Reset(const char* stream, unsigned length) {
  Utf8DecoderBase::Reset(buffer_,
                         kBufferSize,
                         reinterpret_cast<const uint8_t*>(stream),
                         length);
}

Project: node
File: unicode.cc

bool ConnectorPunctuation::Is(uchar c) {
  int chunk_index = c >> 13;
  switch (chunk_index) {
    case 0: return LookupPredicate(kConnectorPunctuationTable0,
                                       kConnectorPunctuationTable0Size,
                                       c);
    case 1: return LookupPredicate(kConnectorPunctuationTable1,
                                       kConnectorPunctuationTable1Size,
                                       c);
    case 7: return LookupPredicate(kConnectorPunctuationTable7,
                                       kConnectorPunctuationTable7Size,
                                       c);
    default: return false;
  }
}

Project: node
File: unicode.cc

void Utf8DecoderBase::WriteUtf16Slow(const uint8_t* stream,
                                     unsigned stream_length,
                                     uint16_t* data,
                                     unsigned data_length) {
  while (data_length != 0) {
    unsigned cursor = 0;

    uint32_t character = Utf8::ValueOf(stream, stream_length, &cursor);
    // There's a total lack of bounds checking for stream
    // as it was already done in Reset.
    stream += cursor;
    stream_length -= cursor;
    if (character > unibrow::Utf16::kMaxNonSurrogateCharCode) {
      *data++ = Utf16::LeadSurrogate(character);
      *data++ = Utf16::TrailSurrogate(character);
      DCHECK(data_length > 1);
      data_length -= 2;
    } else {
      *data++ = character;
      data_length -= 1;
    }
  }
  DCHECK(stream_length >= 0);
}

Project: node
File: unicode.cc

bool LineTerminator::Is(uchar c) {
  int chunk_index = c >> 13;
  switch (chunk_index) {
    case 0: return LookupPredicate(kLineTerminatorTable0,
                                       kLineTerminatorTable0Size,
                                       c);
    case 1: return LookupPredicate(kLineTerminatorTable1,
                                       kLineTerminatorTable1Size,
                                       c);
    default: return false;
  }
}

Project: node
File: unicode.cc

int Ecma262UnCanonicalize::Convert(uchar c,
                      uchar n,
                      uchar* result,
                      bool* allow_caching_ptr) {
  int chunk_index = c >> 13;
  switch (chunk_index) {
    case 0: return LookupMapping<true>(kEcma262UnCanonicalizeTable0,
                                           kEcma262UnCanonicalizeTable0Size,
                                           kEcma262UnCanonicalizeMultiStrings0,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 1: return LookupMapping<true>(kEcma262UnCanonicalizeTable1,
                                           kEcma262UnCanonicalizeTable1Size,
                                           kEcma262UnCanonicalizeMultiStrings1,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 5: return LookupMapping<true>(kEcma262UnCanonicalizeTable5,
                                           kEcma262UnCanonicalizeTable5Size,
                                           kEcma262UnCanonicalizeMultiStrings5,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 7: return LookupMapping<true>(kEcma262UnCanonicalizeTable7,
                                           kEcma262UnCanonicalizeTable7Size,
                                           kEcma262UnCanonicalizeMultiStrings7,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    default: return 0;
  }
}

Project: node
File: unicode-inl.h

template <class T, int s> int Mapping<T, s>::get(uchar c, uchar n,
    uchar* result) {
  CacheEntry entry = entries_[c & kMask];
  if (entry.code_point_ == c) {
    if (entry.offset_ == 0) {
      return 0;
    } else {
      result[0] = c + entry.offset_;
      return 1;
    }
  } else {
    return CalculateValue(c, n, result);
  }
}

Project: node
File: unicode-inl.h

unsigned Utf8Decoder<kBufferSize>::WriteUtf16(uint16_t* data,
                                              unsigned length) const {
  DCHECK(length > 0);
  if (length > utf16_length_) length = utf16_length_;
  // memcpy everything in buffer.
  unsigned buffer_length =
      last_byte_of_buffer_unused_ ? kBufferSize - 1 : kBufferSize;
  unsigned memcpy_length = length <= buffer_length ? length : buffer_length;
  v8::internal::MemCopy(data, buffer_, memcpy_length * sizeof(uint16_t));
  if (length <= buffer_length) return length;
  DCHECK(unbuffered_start_ != NULL);
  // Copy the rest the slow way.
  WriteUtf16Slow(unbuffered_start_, unbuffered_length_, data + buffer_length,
                 length - buffer_length);
  return length;
}

Project: node
File: unicode.cc

bool CombiningMark::Is(uchar c) {
  int chunk_index = c >> 13;
  switch (chunk_index) {
    case 0: return LookupPredicate(kCombiningMarkTable0,
                                       kCombiningMarkTable0Size,
                                       c);
    case 1: return LookupPredicate(kCombiningMarkTable1,
                                       kCombiningMarkTable1Size,
                                       c);
    case 5: return LookupPredicate(kCombiningMarkTable5,
                                       kCombiningMarkTable5Size,
                                       c);
    case 7: return LookupPredicate(kCombiningMarkTable7,
                                       kCombiningMarkTable7Size,
                                       c);
    default: return false;
  }
}

Project: node
File: unicode-inl.h

template <class T, int s> bool Predicate<T, s>::CalculateValue(
    uchar code_point) {
  bool result = T::Is(code_point);
  entries_[code_point & kMask] = CacheEntry(code_point, result);
  return result;
}

Project: node
File: unicode-inl.h

template <class T, int s> int Mapping<T, s>::CalculateValue(uchar c, uchar n,
    uchar* result) {
  bool allow_caching = true;
  int length = T::Convert(c, n, result, &allow_caching);
  if (allow_caching) {
    if (length == 1) {
      entries_[c & kMask] = CacheEntry(c, result[0] - c);
      return 1;
    } else {
      entries_[c & kMask] = CacheEntry(c, 0);
      return 0;
    }
  } else {
    return length;
  }
}

Project: node
File: unicode.cc

bool Letter::Is(uchar c) {
  int chunk_index = c >> 13;
  switch (chunk_index) {
    case 0: return LookupPredicate(kLetterTable0,
                                       kLetterTable0Size,
                                       c);
    case 1: return LookupPredicate(kLetterTable1,
                                       kLetterTable1Size,
                                       c);
    case 2: return LookupPredicate(kLetterTable2,
                                       kLetterTable2Size,
                                       c);
    case 3: return LookupPredicate(kLetterTable3,
                                       kLetterTable3Size,
                                       c);
    case 4: return LookupPredicate(kLetterTable4,
                                       kLetterTable4Size,
                                       c);
    case 5: return LookupPredicate(kLetterTable5,
                                       kLetterTable5Size,
                                       c);
    case 6: return LookupPredicate(kLetterTable6,
                                       kLetterTable6Size,
                                       c);
    case 7: return LookupPredicate(kLetterTable7,
                                       kLetterTable7Size,
                                       c);
    default: return false;
  }
}

Project: node
File: unicode.cc

bool WhiteSpace::Is(uchar c) {
  int chunk_index = c >> 13;
  switch (chunk_index) {
    case 0: return LookupPredicate(kWhiteSpaceTable0,
                                       kWhiteSpaceTable0Size,
                                       c);
    case 1: return LookupPredicate(kWhiteSpaceTable1,
                                       kWhiteSpaceTable1Size,
                                       c);
    default: return false;
  }
}

Project: node
File: unicode.cc

int CanonicalizationRange::Convert(uchar c,
                      uchar n,
                      uchar* result,
                      bool* allow_caching_ptr) {
  int chunk_index = c >> 13;
  switch (chunk_index) {
    case 0: return LookupMapping<false>(kCanonicalizationRangeTable0,
                                           kCanonicalizationRangeTable0Size,
                                           kCanonicalizationRangeMultiStrings0,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 1: return LookupMapping<false>(kCanonicalizationRangeTable1,
                                           kCanonicalizationRangeTable1Size,
                                           kCanonicalizationRangeMultiStrings1,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 7: return LookupMapping<false>(kCanonicalizationRangeTable7,
                                           kCanonicalizationRangeTable7Size,
                                           kCanonicalizationRangeMultiStrings7,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    default: return 0;
  }
}

Project: node
File: unicode-inl.h

uint16_t Latin1::ConvertNonLatin1ToLatin1(uint16_t c) {
  DCHECK(c > Latin1::kMaxChar);
  switch (c) {
    // This are equivalent characters in unicode.
    case 0x39c:
    case 0x3bc:
      return 0xb5;
    // This is an uppercase of a Latin-1 character
    // outside of Latin-1.
    case 0x178:
      return 0xff;
  }
  return 0;
}

Project: node
File: unicode.cc

static inline uchar GetEntry(int32_t entry) {
  return entry & (kStartBit - 1);
}

Project: node
File: unicode.cc

uchar Utf8::CalculateValue(const byte* str,
                           unsigned length,
                           unsigned* cursor) {
  // We only get called for non-ASCII characters.
  if (length == 1) {
    *cursor += 1;
    return kBadChar;
  }
  byte first = str[0];
  byte second = str[1] ^ 0x80;
  if (second & 0xC0) {
    *cursor += 1;
    return kBadChar;
  }
  if (first < 0xE0) {
    if (first < 0xC0) {
      *cursor += 1;
      return kBadChar;
    }
    uchar code_point = ((first << 6) | second) & kMaxTwoByteChar;
    if (code_point <= kMaxOneByteChar) {
      *cursor += 1;
      return kBadChar;
    }
    *cursor += 2;
    return code_point;
  }
  if (length == 2) {
    *cursor += 1;
    return kBadChar;
  }
  byte third = str[2] ^ 0x80;
  if (third & 0xC0) {
    *cursor += 1;
    return kBadChar;
  }
  if (first < 0xF0) {
    uchar code_point = ((((first << 6) | second) << 6) | third)
        & kMaxThreeByteChar;
    if (code_point <= kMaxTwoByteChar) {
      *cursor += 1;
      return kBadChar;
    }
    *cursor += 3;
    return code_point;
  }
  if (length == 3) {
    *cursor += 1;
    return kBadChar;
  }
  byte fourth = str[3] ^ 0x80;
  if (fourth & 0xC0) {
    *cursor += 1;
    return kBadChar;
  }
  if (first < 0xF8) {
    uchar code_point = (((((first << 6 | second) << 6) | third) << 6) | fourth)
        & kMaxFourByteChar;
    if (code_point <= kMaxThreeByteChar) {
      *cursor += 1;
      return kBadChar;
    }
    *cursor += 4;
    return code_point;
  }
  *cursor += 1;
  return kBadChar;
}

Project: node
File: unicode-inl.h

Utf8DecoderBase::Utf8DecoderBase(uint16_t* buffer,
                                 unsigned buffer_length,
                                 const uint8_t* stream,
                                 unsigned stream_length) {
  Reset(buffer, buffer_length, stream, stream_length);
}

Project: node
File: unicode-inl.h

uchar Utf8::ValueOf(const byte* bytes, unsigned length, unsigned* cursor) {
  if (length <= 0) return kBadChar;
  byte first = bytes[0];
  // Characters between 0000 and 0007F are encoded as a single character
  if (first <= kMaxOneByteChar) {
    *cursor += 1;
    return first;
  }
  return CalculateValue(bytes, length, cursor);
}

Project: node
File: unicode.cc

int UnicodeData::GetByteCount() {
  return kUppercaseTable0Size * sizeof(int32_t)  // NOLINT
      + kUppercaseTable1Size * sizeof(int32_t)  // NOLINT
      + kUppercaseTable5Size * sizeof(int32_t)  // NOLINT
      + kUppercaseTable7Size * sizeof(int32_t)  // NOLINT
      + kLowercaseTable0Size * sizeof(int32_t)  // NOLINT
      + kLowercaseTable1Size * sizeof(int32_t)  // NOLINT
      + kLowercaseTable5Size * sizeof(int32_t)  // NOLINT
      + kLowercaseTable7Size * sizeof(int32_t)  // NOLINT
      + kLetterTable0Size * sizeof(int32_t)  // NOLINT
      + kLetterTable1Size * sizeof(int32_t)  // NOLINT
      + kLetterTable2Size * sizeof(int32_t)  // NOLINT
      + kLetterTable3Size * sizeof(int32_t)  // NOLINT
      + kLetterTable4Size * sizeof(int32_t)  // NOLINT
      + kLetterTable5Size * sizeof(int32_t)  // NOLINT
      + kLetterTable6Size * sizeof(int32_t)  // NOLINT
      + kLetterTable7Size * sizeof(int32_t)  // NOLINT
      + kNumberTable0Size * sizeof(int32_t)  // NOLINT
      + kNumberTable5Size * sizeof(int32_t)  // NOLINT
      + kNumberTable7Size * sizeof(int32_t)  // NOLINT
      + kWhiteSpaceTable0Size * sizeof(int32_t)  // NOLINT
      + kWhiteSpaceTable1Size * sizeof(int32_t)  // NOLINT
      + kLineTerminatorTable0Size * sizeof(int32_t)  // NOLINT
      + kLineTerminatorTable1Size * sizeof(int32_t)  // NOLINT
      + kCombiningMarkTable0Size * sizeof(int32_t)  // NOLINT
      + kCombiningMarkTable1Size * sizeof(int32_t)  // NOLINT
      + kCombiningMarkTable5Size * sizeof(int32_t)  // NOLINT
      + kCombiningMarkTable7Size * sizeof(int32_t)  // NOLINT
      + kConnectorPunctuationTable0Size * sizeof(int32_t)  // NOLINT
      + kConnectorPunctuationTable1Size * sizeof(int32_t)  // NOLINT
      + kConnectorPunctuationTable7Size * sizeof(int32_t)  // NOLINT
      + kToLowercaseMultiStrings0Size * sizeof(MultiCharacterSpecialCase<2>)  // NOLINT
      + kToLowercaseMultiStrings1Size * sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
      + kToLowercaseMultiStrings5Size * sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
      + kToLowercaseMultiStrings7Size * sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
      + kToUppercaseMultiStrings0Size * sizeof(MultiCharacterSpecialCase<3>)  // NOLINT
      + kToUppercaseMultiStrings1Size * sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
      + kToUppercaseMultiStrings5Size * sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
      + kToUppercaseMultiStrings7Size * sizeof(MultiCharacterSpecialCase<3>)  // NOLINT
      + kEcma262CanonicalizeMultiStrings0Size * sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
      + kEcma262CanonicalizeMultiStrings1Size * sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
      + kEcma262CanonicalizeMultiStrings5Size * sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
      + kEcma262CanonicalizeMultiStrings7Size * sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
      + kEcma262UnCanonicalizeMultiStrings0Size * sizeof(MultiCharacterSpecialCase<4>)  // NOLINT
      + kEcma262UnCanonicalizeMultiStrings1Size * sizeof(MultiCharacterSpecialCase<2>)  // NOLINT
      + kEcma262UnCanonicalizeMultiStrings5Size * sizeof(MultiCharacterSpecialCase<2>)  // NOLINT
      + kEcma262UnCanonicalizeMultiStrings7Size * sizeof(MultiCharacterSpecialCase<2>)  // NOLINT
      + kCanonicalizationRangeMultiStrings0Size * sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
      + kCanonicalizationRangeMultiStrings1Size * sizeof(MultiCharacterSpecialCase<1>)  // NOLINT
      + kCanonicalizationRangeMultiStrings7Size * sizeof(MultiCharacterSpecialCase<1>); // NOLINT
}

Project: node
File: unicode-inl.h

unsigned Utf8::Length(uchar c, int previous) {
  if (c <= kMaxOneByteChar) {
    return 1;
  } else if (c <= kMaxTwoByteChar) {
    return 2;
  } else if (c <= kMaxThreeByteChar) {
    if (Utf16::IsTrailSurrogate(c) &&
        Utf16::IsLeadSurrogate(previous)) {
      return kSizeOfUnmatchedSurrogate - kBytesSavedByCombiningSurrogates;
    }
    return 3;
  } else {
    return 4;
  }
}

Project: node
File: unicode.cc

bool Lowercase::Is(uchar c) {
  int chunk_index = c >> 13;
  switch (chunk_index) {
    case 0: return LookupPredicate(kLowercaseTable0,
                                       kLowercaseTable0Size,
                                       c);
    case 1: return LookupPredicate(kLowercaseTable1,
                                       kLowercaseTable1Size,
                                       c);
    case 5: return LookupPredicate(kLowercaseTable5,
                                       kLowercaseTable5Size,
                                       c);
    case 7: return LookupPredicate(kLowercaseTable7,
                                       kLowercaseTable7Size,
                                       c);
    default: return false;
  }
}

Project: node
File: unicode.cc

static bool LookupPredicate(const int32_t* table, uint16_t size, uchar chr) {
  static const int kEntryDist = 1;
  uint16_t value = chr & (kChunkBits - 1);
  unsigned int low = 0;
  unsigned int high = size - 1;
  while (high != low) {
    unsigned int mid = low + ((high - low) >> 1);
    uchar current_value = GetEntry(TableGet<kEntryDist>(table, mid));
    // If we've found an entry less than or equal to this one, and the
    // next one is not also less than this one, we've arrived.
    if ((current_value <= value) &&
        (mid + 1 == size ||
         GetEntry(TableGet<kEntryDist>(table, mid + 1)) > value)) {
      low = mid;
      break;
    } else if (current_value < value) {
      low = mid + 1;
    } else if (current_value > value) {
      // If we've just checked the bottom-most value and it's not
      // the one we're looking for, we're done.
      if (mid == 0) break;
      high = mid - 1;
    }
  }
  int32_t field = TableGet<kEntryDist>(table, low);
  uchar entry = GetEntry(field);
  bool is_start = IsStart(field);
  return (entry == value) || (entry < value && is_start);
}

Project: node
File: unicode.cc

void Utf8DecoderBase::Reset(uint16_t* buffer,
                            unsigned buffer_length,
                            const uint8_t* stream,
                            unsigned stream_length) {
  // Assume everything will fit in the buffer and stream won't be needed.
  last_byte_of_buffer_unused_ = false;
  unbuffered_start_ = NULL;
  unbuffered_length_ = 0;
  bool writing_to_buffer = true;
  // Loop until stream is read, writing to buffer as long as buffer has space.
  unsigned utf16_length = 0;
  while (stream_length != 0) {
    unsigned cursor = 0;
    uint32_t character = Utf8::ValueOf(stream, stream_length, &cursor);
    DCHECK(cursor > 0 && cursor <= stream_length);
    stream += cursor;
    stream_length -= cursor;
    bool is_two_characters = character > Utf16::kMaxNonSurrogateCharCode;
    utf16_length += is_two_characters ? 2 : 1;
    // Don't need to write to the buffer, but still need utf16_length.
    if (!writing_to_buffer) continue;
    // Write out the characters to the buffer.
    // Must check for equality with buffer_length as we've already updated it.
    if (utf16_length <= buffer_length) {
      if (is_two_characters) {
        *buffer++ = Utf16::LeadSurrogate(character);
        *buffer++ = Utf16::TrailSurrogate(character);
      } else {
        *buffer++ = character;
      }
      if (utf16_length == buffer_length) {
        // Just wrote last character of buffer
        writing_to_buffer = false;
        unbuffered_start_ = stream;
        unbuffered_length_ = stream_length;
      }
      continue;
    }
    // Have gone over buffer.
    // Last char of buffer is unused, set cursor back.
    DCHECK(is_two_characters);
    writing_to_buffer = false;
    last_byte_of_buffer_unused_ = true;
    unbuffered_start_ = stream - cursor;
    unbuffered_length_ = stream_length + cursor;
  }
  utf16_length_ = utf16_length;
}

Project: node
File: unicode-inl.h

template <class T, int s> bool Predicate<T, s>::get(uchar code_point) {
  CacheEntry entry = entries_[code_point & kMask];
  if (entry.code_point_ == code_point) return entry.value_;
  return CalculateValue(code_point);
}

Project: node
File: unicode-inl.h

unsigned Utf8::EncodeOneByte(char* str, uint8_t c) {
  static const int kMask = ~(1 << 6);
  if (c <= kMaxOneByteChar) {
    str[0] = c;
    return 1;
  }
  str[0] = 0xC0 | (c >> 6);
  str[1] = 0x80 | (c & kMask);
  return 2;
}

Project: node
File: unicode.cc

static inline uchar TableGet(const int32_t* table, int index) {
  return table[D * index];
}

Project: node
File: unicode.cc

int ToUppercase::Convert(uchar c,
                      uchar n,
                      uchar* result,
                      bool* allow_caching_ptr) {
  int chunk_index = c >> 13;
  switch (chunk_index) {
    case 0: return LookupMapping<true>(kToUppercaseTable0,
                                           kToUppercaseTable0Size,
                                           kToUppercaseMultiStrings0,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 1: return LookupMapping<true>(kToUppercaseTable1,
                                           kToUppercaseTable1Size,
                                           kToUppercaseMultiStrings1,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 5: return LookupMapping<true>(kToUppercaseTable5,
                                           kToUppercaseTable5Size,
                                           kToUppercaseMultiStrings5,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    case 7: return LookupMapping<true>(kToUppercaseTable7,
                                           kToUppercaseTable7Size,
                                           kToUppercaseMultiStrings7,
                                           c,
                                           n,
                                           result,
                                           allow_caching_ptr);
    default: return 0;
  }
}

Project: node
File: unicode.cc

bool Uppercase::Is(uchar c) {
  int chunk_index = c >> 13;
  switch (chunk_index) {
    case 0: return LookupPredicate(kUppercaseTable0,
                                       kUppercaseTable0Size,
                                       c);
    case 1: return LookupPredicate(kUppercaseTable1,
                                       kUppercaseTable1Size,
                                       c);
    case 5: return LookupPredicate(kUppercaseTable5,
                                       kUppercaseTable5Size,
                                       c);
    case 7: return LookupPredicate(kUppercaseTable7,
                                       kUppercaseTable7Size,
                                       c);
    default: return false;
  }
}

Project: node
File: unicode.cc

bool Number::Is(uchar c) {
  int chunk_index = c >> 13;
  switch (chunk_index) {
    case 0: return LookupPredicate(kNumberTable0,
                                       kNumberTable0Size,
                                       c);
    case 5: return LookupPredicate(kNumberTable5,
                                       kNumberTable5Size,
                                       c);
    case 7: return LookupPredicate(kNumberTable7,
                                       kNumberTable7Size,
                                       c);
    default: return false;
  }
}