sdk/lib/utf/utf.dart

// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/**
 * Support for encoding and decoding Unicode characters in UTF-8, UTF-16, and 
 * UTF-32.
 */
library dart.utf;
import "dart:async";
import "dart:collection";
part "utf_stream.dart";
part "utf8.dart";
part "utf16.dart";
part "utf32.dart";

// TODO(jmesserly): would be nice to have this on String (dartbug.com/6501).
/**
 * Provide a list of Unicode codepoints for a given string.
 */
List<int> stringToCodepoints(String str) {
  // Note: str.codeUnits gives us 16-bit code units on all Dart implementations.
  // So we need to convert.
  return _utf16CodeUnitsToCodepoints(str.codeUnits);
}

/**
 * Generate a string from the provided Unicode codepoints.
 *
 * *Deprecated* Use [String.fromCharCodes] instead.
 */
String codepointsToString(List<int> codepoints) {
  return new String.fromCharCodes(codepoints);
}

/**
 * Invalid codepoints or encodings may be substituted with the value U+fffd.
 */
const int UNICODE_REPLACEMENT_CHARACTER_CODEPOINT = 0xfffd;
const int UNICODE_BOM = 0xfeff;
const int UNICODE_UTF_BOM_LO = 0xff;
const int UNICODE_UTF_BOM_HI = 0xfe;

const int UNICODE_BYTE_ZERO_MASK = 0xff;
const int UNICODE_BYTE_ONE_MASK = 0xff00;
const int UNICODE_VALID_RANGE_MAX = 0x10ffff;
const int UNICODE_PLANE_ONE_MAX = 0xffff;
const int UNICODE_UTF16_RESERVED_LO = 0xd800;
const int UNICODE_UTF16_RESERVED_HI = 0xdfff;
const int UNICODE_UTF16_OFFSET = 0x10000;
const int UNICODE_UTF16_SURROGATE_UNIT_0_BASE = 0xd800;
const int UNICODE_UTF16_SURROGATE_UNIT_1_BASE = 0xdc00;
const int UNICODE_UTF16_HI_MASK = 0xffc00;
const int UNICODE_UTF16_LO_MASK = 0x3ff;

/**
 * Encode code points as UTF16 code units.
 */
List<int> _codepointsToUtf16CodeUnits(
    List<int> codepoints,
    [int offset = 0,
     int length,
     int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {

  _ListRange listRange = new _ListRange(codepoints, offset, length);
  int encodedLength = 0;
  for (int value in listRange) {
    if ((value >= 0 && value < UNICODE_UTF16_RESERVED_LO) ||
        (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) {
      encodedLength++;
    } else if (value > UNICODE_PLANE_ONE_MAX &&
        value <= UNICODE_VALID_RANGE_MAX) {
      encodedLength += 2;
    } else {
      encodedLength++;
    }
  }

  List<int> codeUnitsBuffer = new List<int>(encodedLength);
  int j = 0;
  for (int value in listRange) {
    if ((value >= 0 && value < UNICODE_UTF16_RESERVED_LO) ||
        (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) {
      codeUnitsBuffer[j++] = value;
    } else if (value > UNICODE_PLANE_ONE_MAX &&
        value <= UNICODE_VALID_RANGE_MAX) {
      int base = value - UNICODE_UTF16_OFFSET;
      codeUnitsBuffer[j++] = UNICODE_UTF16_SURROGATE_UNIT_0_BASE +
          ((base & UNICODE_UTF16_HI_MASK) >> 10);
      codeUnitsBuffer[j++] = UNICODE_UTF16_SURROGATE_UNIT_1_BASE +
          (base & UNICODE_UTF16_LO_MASK);
    } else if (replacementCodepoint != null) {
      codeUnitsBuffer[j++] = replacementCodepoint;
    } else {
      throw new ArgumentError("Invalid encoding");
    }
  }
  return codeUnitsBuffer;
}

/**
 * Decodes the utf16 codeunits to codepoints.
 */
List<int> _utf16CodeUnitsToCodepoints(
    List<int> utf16CodeUnits, [int offset = 0, int length,
    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
  _ListRangeIterator source =
      (new _ListRange(utf16CodeUnits, offset, length)).iterator;
  Utf16CodeUnitDecoder decoder = new Utf16CodeUnitDecoder
      .fromListRangeIterator(source, replacementCodepoint);
  List<int> codepoints = new List<int>(source.remaining);
  int i = 0;
  while (decoder.moveNext()) {
    codepoints[i++] = decoder.current;
  }
  if (i == codepoints.length) {
    return codepoints;
  } else {
    List<int> codepointTrunc = new List<int>(i);
    codepointTrunc.setRange(0, i, codepoints);
    return codepointTrunc;
  }
}

/**
 * An Iterator<int> of codepoints built on an Iterator of UTF-16 code units.
 * The parameters can override the default Unicode replacement character. Set
 * the replacementCharacter to null to throw an ArgumentError
 * rather than replace the bad value.
 */
class Utf16CodeUnitDecoder implements Iterator<int> {
  final _ListRangeIterator utf16CodeUnitIterator;
  final int replacementCodepoint;
  int _current = null;

  Utf16CodeUnitDecoder(List<int> utf16CodeUnits, [int offset = 0, int length,
      int this.replacementCodepoint =
      UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) :
      utf16CodeUnitIterator =
          (new _ListRange(utf16CodeUnits, offset, length)).iterator;

  Utf16CodeUnitDecoder.fromListRangeIterator(
      _ListRangeIterator this.utf16CodeUnitIterator,
      int this.replacementCodepoint);

  Iterator<int> get iterator => this;

  int get current => _current;

  bool moveNext() {
    _current = null;
    if (!utf16CodeUnitIterator.moveNext()) return false;

    int value = utf16CodeUnitIterator.current;
    if (value < 0) {
      if (replacementCodepoint != null) {
        _current = replacementCodepoint;
      } else {
        throw new ArgumentError(
            "Invalid UTF16 at ${utf16CodeUnitIterator.position}");
      }
    } else if (value < UNICODE_UTF16_RESERVED_LO ||
        (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) {
      // transfer directly
      _current = value;
    } else if (value < UNICODE_UTF16_SURROGATE_UNIT_1_BASE &&
        utf16CodeUnitIterator.moveNext()) {
      // merge surrogate pair
      int nextValue = utf16CodeUnitIterator.current;
      if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_1_BASE &&
          nextValue <= UNICODE_UTF16_RESERVED_HI) {
        value = (value - UNICODE_UTF16_SURROGATE_UNIT_0_BASE) << 10;
        value += UNICODE_UTF16_OFFSET +
            (nextValue - UNICODE_UTF16_SURROGATE_UNIT_1_BASE);
        _current = value;
      } else {
        if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_0_BASE &&
           nextValue < UNICODE_UTF16_SURROGATE_UNIT_1_BASE) {
          utf16CodeUnitIterator.backup();
        }
        if (replacementCodepoint != null) {
          _current = replacementCodepoint;
        } else {
          throw new ArgumentError(
              "Invalid UTF16 at ${utf16CodeUnitIterator.position}");
        }
      }
    } else if (replacementCodepoint != null) {
      _current = replacementCodepoint;
    } else {
      throw new ArgumentError(
          "Invalid UTF16 at ${utf16CodeUnitIterator.position}");
    }
    return true;
  }
}

/**
 * _ListRange in an internal type used to create a lightweight Interable on a
 * range within a source list. DO NOT MODIFY the underlying list while
 * iterating over it. The results of doing so are undefined.
 */
// TODO(floitsch): Consider removing the extend and switch to implements since
// that's cheaper to allocate.
class _ListRange extends IterableBase {
  final List _source;
  final int _offset;
  final int _length;

  _ListRange(source, [offset = 0, length]) :
      this._source = source,
      this._offset = offset,
      this._length = (length == null ? source.length - offset : length) {
    if (_offset < 0 || _offset > _source.length) {
      throw new RangeError.value(_offset);
    }
    if (_length != null && (_length < 0)) {
      throw new RangeError.value(_length);
    }
    if (_length + _offset > _source.length) {
      throw new RangeError.value(_length + _offset);
    }
  }

  _ListRangeIterator get iterator =>
      new _ListRangeIteratorImpl(_source, _offset, _offset + _length);

  int get length => _length;
}

/**
 * The _ListRangeIterator provides more capabilities than a standard iterator,
 * including the ability to get the current position, count remaining items,
 * and move forward/backward within the iterator.
 */
abstract class _ListRangeIterator implements Iterator<int> {
  bool moveNext();
  int get current;
  int get position;
  void backup([by]);
  int get remaining;
  void skip([count]);
}

class _ListRangeIteratorImpl implements _ListRangeIterator {
  final List<int> _source;
  int _offset;
  final int _end;

  _ListRangeIteratorImpl(this._source, int offset, this._end)
      : _offset = offset - 1;

  int get current => _source[_offset];

  bool moveNext() => ++_offset < _end;

  int get position => _offset;

  void backup([int by = 1]) {
    _offset -= by;
  }

  int get remaining => _end - _offset - 1;

  void skip([int count = 1]) {
    _offset += count;
  }
}