blob: bd7da5755a93626812f7f116c553661efcc1acfc [file] [log] [blame]
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "platform/unicode.h"
#include "vm/allocation.h"
#include "vm/globals.h"
#include "vm/object.h"
namespace dart {
// A constant mask that can be 'and'ed with a word of data to determine if it
// is all ASCII (with no Latin1 characters).
#if defined(ARCH_IS_64_BIT)
static constexpr uintptr_t kAsciiWordMask = DART_UINT64_C(0x8080808080808080);
#else
static constexpr uintptr_t kAsciiWordMask = 0x80808080u;
#endif
intptr_t Utf8::Length(const String& str) {
if (str.IsOneByteString()) {
// For 1-byte strings, all code points < 0x80 have single-byte UTF-8
// encodings and all >= 0x80 have two-byte encodings. To get the length,
// start with the number of code points and add the number of high bits in
// the bytes.
uintptr_t char_length = str.Length();
uintptr_t length = char_length;
NoSafepointScope no_safepoint;
const uintptr_t* data =
reinterpret_cast<const uintptr_t*>(OneByteString::DataStart(str));
uintptr_t i;
for (i = sizeof(uintptr_t); i <= char_length; i += sizeof(uintptr_t)) {
uintptr_t chunk = *data++;
chunk &= kAsciiWordMask;
if (chunk != 0) {
// Shuffle the bits until we have a count of bits in the low nibble.
#if defined(ARCH_IS_64_BIT)
chunk += chunk >> 32;
#endif
chunk += chunk >> 16;
chunk += chunk >> 8;
length += (chunk >> 7) & 0xf;
}
}
// Take care of the tail of the string, the last length % wordsize chars.
i -= sizeof(uintptr_t);
for (; i < char_length; i++) {
if (str.CharAt(i) > kMaxOneByteChar) length++;
}
return length;
}
// Slow case for 2-byte strings that handles surrogate pairs and longer UTF-8
// encodings.
intptr_t length = 0;
String::CodePointIterator it(str);
while (it.Next()) {
int32_t ch = it.Current();
length += Utf8::Length(ch);
}
return length;
}
intptr_t Utf8::Encode(const String& src, char* dst, intptr_t len) {
uintptr_t array_len = len;
intptr_t pos = 0;
ASSERT(static_cast<intptr_t>(array_len) >= Length(src));
if (src.IsOneByteString()) {
// For 1-byte strings, all code points < 0x80 have single-byte UTF-8
// encodings and all >= 0x80 have two-byte encodings.
NoSafepointScope scope;
const uintptr_t* data =
reinterpret_cast<const uintptr_t*>(OneByteString::DataStart(src));
uintptr_t char_length = src.Length();
uintptr_t pos = 0;
ASSERT(kMaxOneByteChar + 1 == 0x80);
for (uintptr_t i = 0; i < char_length; i += sizeof(uintptr_t)) {
// Read the input one word at a time and just write it verbatim if it is
// plain ASCII, as determined by the mask.
if (i + sizeof(uintptr_t) <= char_length &&
(*data & kAsciiWordMask) == 0 &&
pos + sizeof(uintptr_t) <= array_len) {
StoreUnaligned(reinterpret_cast<uintptr_t*>(dst + pos), *data);
pos += sizeof(uintptr_t);
} else {
// Process up to one word of input that contains non-ASCII Latin1
// characters.
const uint8_t* p = reinterpret_cast<const uint8_t*>(data);
const uint8_t* limit =
Utils::Minimum(p + sizeof(uintptr_t), p + (char_length - i));
for (; p < limit; p++) {
uint8_t c = *p;
// These calls to Length and Encode get inlined and the cases for 3
// and 4 byte sequences are removed.
intptr_t bytes = Length(c);
if (pos + bytes > array_len) {
return pos;
}
Encode(c, reinterpret_cast<char*>(dst) + pos);
pos += bytes;
}
}
data++;
}
} else {
// For two-byte strings, which can contain 3 and 4-byte UTF-8 encodings,
// which can result in surrogate pairs, use the more general code.
String::CodePointIterator it(src);
while (it.Next()) {
int32_t ch = it.Current();
ASSERT(!Utf::IsOutOfRange(ch));
if (Utf16::IsSurrogate(ch)) {
// Encode unpaired surrogates as replacement characters to ensure the
// output is valid UTF-8. Encoded size is the same (3), so the computed
// length is still valid.
ch = Utf::kReplacementChar;
}
intptr_t num_bytes = Utf8::Length(ch);
if (pos + num_bytes > len) {
break;
}
Utf8::Encode(ch, &dst[pos]);
pos += num_bytes;
}
}
return pos;
}
} // namespace dart