runtime/vm/unicode.cc

// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

#include "vm/unicode.h"

#include "vm/allocation.h"
#include "vm/globals.h"
#include "vm/object.h"

namespace dart {

static const uint8_t kTrailBytes[256] = {
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
  4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0
};


static const uint32_t kMagicBits[7] = {
  0,  // padding
  0x00000000,
  0x00003080,
  0x000E2080,
  0x03C82080,
  0xFA082080,
  0x82082080
};


// Minimum values of code points used to check shortest form.
static const uint32_t kOverlongMinimum[7] = {
  0,  // padding
  0x0,
  0x80,
  0x800,
  0x10000,
  0xFFFFFFFF,
  0xFFFFFFFF
};


static bool IsTrailByte(uint8_t code_unit) {
  return (code_unit & 0xc0) == 0x80;
}


// Returns true if the code point is a high- or low-surrogate.
static bool IsSurrogate(uint32_t code_point) {
  return (code_point & 0xfffff800) == 0xd800;
}


// Returns true if the code point value is above Plane 17.
static bool IsOutOfRange(uint32_t code_point) {
  return code_point > 0x10FFFF;
}


// Returns true if the byte sequence is ill-formed.
static bool IsNonShortestForm(uint32_t code_point, size_t num_bytes) {
  return code_point < kOverlongMinimum[num_bytes];
}


// Returns a count of the number of UTF-8 trail bytes.
intptr_t Utf8::CodePointCount(const char* str, intptr_t* width) {
  bool is_two_byte_string = false;
  bool is_four_byte_string = false;
  intptr_t len = 0;
  for (; *str != '\0'; ++str) {
    uint8_t code_unit = *str;
    if (!IsTrailByte(code_unit)) {
      ++len;
    }
    if (code_unit > 0xC3) {  // > U+00FF
      if (code_unit < 0xF0) {  // < U+10000
        is_two_byte_string = true;
      } else {
        is_four_byte_string = true;
      }
    }
  }
  if (is_four_byte_string) {
    *width = 4;
  } else if (is_two_byte_string) {
    *width = 2;
  } else {
    *width = 1;
  }
  return len;
}


// Returns true if str is a valid NUL-terminated UTF-8 string.
bool Utf8::IsValid(const char* str) {
  intptr_t i = 0;
  while (str[i] != '\0') {
    uint32_t ch = str[i] & 0xFF;
    intptr_t j = 1;
    if (ch >= 0x80) {
      uint8_t num_trail_bytes = kTrailBytes[ch];
      bool is_malformed = false;
      for (; j < num_trail_bytes; ++j) {
        if (str[i + j] != '\0') {
          uint8_t code_unit = str[i + j];
          is_malformed |= !IsTrailByte(code_unit);
          ch = (ch << 6) + code_unit;
        } else {
          return false;
        }
      }
      ch -= kMagicBits[num_trail_bytes];
      if (!((is_malformed == false) &&
            (j == num_trail_bytes) &&
            !IsOutOfRange(ch) &&
            !IsNonShortestForm(ch, j) &&
            !IsSurrogate(ch))) {
        return false;
      }
    }
    i += j;
  }
  return true;
}


intptr_t Utf8::Length(int32_t ch) {
  if (ch <= kMaxOneByteChar) {
    return 1;
  } else if (ch <= kMaxTwoByteChar) {
    return 2;
  } else if (ch <= kMaxThreeByteChar) {
    return 3;
  }
  ASSERT(ch <= kMaxFourByteChar);
  return 4;
}


intptr_t Utf8::Length(const String& str) {
  intptr_t length = 0;
  for (intptr_t i = 0; i < str.Length(); ++i) {
    int32_t ch = str.CharAt(i);
    length += Utf8::Length(ch);
  }
  return length;
}


intptr_t Utf8::Encode(int32_t ch, char* dst) {
  static const int kMask = ~(1 << 6);
  if (ch <= kMaxOneByteChar) {
    dst[0] = ch;
    return 1;
  }
  if (ch <= kMaxTwoByteChar) {
    dst[0] = 0xC0 | (ch >> 6);
    dst[1] = 0x80 | (ch & kMask);
    return 2;
  }
  if (ch <= kMaxThreeByteChar) {
    dst[0] = 0xE0 | (ch >> 12);
    dst[1] = 0x80 | ((ch >> 6) & kMask);
    dst[2] = 0x80 | (ch & kMask);
    return 3;
  }
  ASSERT(ch <= kMaxFourByteChar);
  dst[0] = 0xF0 | (ch >> 18);
  dst[1] = 0x80 | ((ch >> 12) & kMask);
  dst[2] = 0x80 | ((ch >> 6) & kMask);
  dst[3] = 0x80 | (ch & kMask);
  return 4;
}


intptr_t Utf8::Encode(const String& src, char* dst, intptr_t len) {
  intptr_t pos = 0;
  for (intptr_t i = 0; i < src.Length(); ++i) {
    intptr_t ch = src.CharAt(i);
    intptr_t num_bytes = Utf8::Length(ch);
    if (pos + num_bytes > len) {
      break;
    }
    Utf8::Encode(ch, &dst[pos]);
    pos += num_bytes;
  }
  return pos;
}


intptr_t Utf8::Decode(const char* src, int32_t* dst) {
  uint32_t ch = src[0] & 0xFF;
  uint32_t i = 1;
  if (ch >= 0x80) {
    uint32_t num_trail_bytes = kTrailBytes[ch];
    bool is_malformed = false;
    for (; i < num_trail_bytes; ++i) {
      if (src[i] != '\0') {
        uint8_t code_unit = src[i];
        is_malformed |= !IsTrailByte(code_unit);
        ch = (ch << 6) + code_unit;
      } else {
        *dst = -1;
        return 0;
      }
    }
    ch -= kMagicBits[num_trail_bytes];
    if (!((is_malformed == false) &&
          (i == num_trail_bytes) &&
          !IsOutOfRange(ch) &&
          !IsNonShortestForm(ch, i) &&
          !IsSurrogate(ch))) {
      *dst = -1;
      return 0;
    }
  }
  *dst = ch;
  return i;
}


template<typename T>
static bool DecodeImpl(const char* src, T* dst, intptr_t len) {
  intptr_t i = 0;
  intptr_t j = 0;
  intptr_t num_bytes;
  for (; src[i] != '\0' && j < len; i += num_bytes, ++j) {
    int32_t ch;
    num_bytes = Utf8::Decode(&src[i], &ch);
    if (ch == -1) {
      return false;  // invalid input
    }
    dst[j] = ch;
  }
  if (src[i] != '\0' && j == len) {
    return false;  // output overflow
  }
  return true;  // success
}


bool Utf8::Decode(const char* src, uint8_t* dst, intptr_t len) {
  return DecodeImpl(src, dst, len);
}


bool Utf8::Decode(const char* src, uint16_t* dst, intptr_t len) {
  return DecodeImpl(src, dst, len);
}


bool Utf8::Decode(const char* src, uint32_t* dst, intptr_t len) {
  return DecodeImpl(src, dst, len);
}

}  // namespace dart