// Copyright (c) 2021, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

#include "vm/object_graph_copy.h"
#include "vm/dart_api_state.h"
#include "vm/flags.h"
#include "vm/heap/weak_table.h"
#include "vm/longjump.h"
#include "vm/object.h"
#include "vm/snapshot.h"
#include "vm/symbols.h"

#define Z zone_

// The list here contains two kinds of classes of objects
//   * objects that will be shared and we will therefore never need to copy
//   * objects that user object graphs should never reference
#define FOR_UNSUPPORTED_CLASSES(V)                                             \
  V(AbstractType)                                                              \
  V(ApiError)                                                                  \
  V(Bool)                                                                      \
  V(CallSiteData)                                                              \
  V(Capability)                                                                \
  V(Class)                                                                     \
  V(ClosureData)                                                               \
  V(Code)                                                                      \
  V(CodeSourceMap)                                                             \
  V(CompressedStackMaps)                                                       \
  V(Context)                                                                   \
  V(ContextScope)                                                              \
  V(DynamicLibrary)                                                            \
  V(Error)                                                                     \
  V(ExceptionHandlers)                                                         \
  V(FfiTrampolineData)                                                         \
  V(Field)                                                                     \
  V(Function)                                                                  \
  V(FunctionType)                                                              \
  V(FutureOr)                                                                  \
  V(ICData)                                                                    \
  V(Instance)                                                                  \
  V(Instructions)                                                              \
  V(InstructionsSection)                                                       \
  V(InstructionsTable)                                                         \
  V(Int32x4)                                                                   \
  V(Integer)                                                                   \
  V(KernelProgramInfo)                                                         \
  V(LanguageError)                                                             \
  V(Library)                                                                   \
  V(LibraryPrefix)                                                             \
  V(LoadingUnit)                                                               \
  V(LocalVarDescriptors)                                                       \
  V(MegamorphicCache)                                                          \
  V(Mint)                                                                      \
  V(MirrorReference)                                                           \
  V(MonomorphicSmiableCall)                                                    \
  V(Namespace)                                                                 \
  V(Number)                                                                    \
  V(ObjectPool)                                                                \
  V(PatchClass)                                                                \
  V(PcDescriptors)                                                             \
  V(Pointer)                                                                   \
  V(ReceivePort)                                                               \
  V(RegExp)                                                                    \
  V(Script)                                                                    \
  V(Sentinel)                                                                  \
  V(SendPort)                                                                  \
  V(SingleTargetCache)                                                         \
  V(Smi)                                                                       \
  V(StackTrace)                                                                \
  V(SubtypeTestCache)                                                          \
  V(Type)                                                                      \
  V(TypeArguments)                                                             \
  V(TypeParameter)                                                             \
  V(TypeParameters)                                                            \
  V(TypeRef)                                                                   \
  V(TypedDataBase)                                                             \
  V(UnhandledException)                                                        \
  V(UnlinkedCall)                                                              \
  V(UnwindError)                                                               \
  V(UserTag)                                                                   \
  V(WeakProperty)                                                              \
  V(WeakSerializationReference)

namespace dart {

DEFINE_FLAG(bool,
            enable_fast_object_copy,
            true,
            "Enable fast path for fast object copy.");
DEFINE_FLAG(bool,
            gc_on_foc_slow_path,
            false,
            "Cause a GC when falling off the fast path for fast object copy.");

const char* kFastAllocationFailed = "fast allocation failed";

struct PtrTypes {
  using Object = ObjectPtr;
  static const dart::UntaggedObject* UntagObject(Object arg) {
    return arg.untag();
  }
  static const dart::ObjectPtr GetObjectPtr(Object arg) { return arg; }
  static const dart::Object& HandlifyObject(ObjectPtr arg) {
    return dart::Object::Handle(arg);
  }

#define DO(V)                                                                  \
  using V = V##Ptr;                                                            \
  static Untagged##V* Untag##V(V##Ptr arg) { return arg.untag(); }             \
  static V##Ptr Get##V##Ptr(V##Ptr arg) { return arg; }                        \
  static V##Ptr Cast##V(ObjectPtr arg) { return dart::V::RawCast(arg); }
  CLASS_LIST_FOR_HANDLES(DO)
#undef DO
};

struct HandleTypes {
  using Object = const Object&;
  static const dart::UntaggedObject* UntagObject(Object arg) {
    return arg.ptr().untag();
  }
  static dart::ObjectPtr GetObjectPtr(Object arg) { return arg.ptr(); }
  static Object HandlifyObject(Object arg) { return arg; }

#define DO(V)                                                                  \
  using V = const V&;                                                          \
  static Untagged##V* Untag##V(V arg) { return arg.ptr().untag(); }            \
  static V##Ptr Get##V##Ptr(V arg) { return arg.ptr(); }                       \
  static V Cast##V(const dart::Object& arg) { return dart::V::Cast(arg); }
  CLASS_LIST_FOR_HANDLES(DO)
#undef DO
};

DART_FORCE_INLINE
static ObjectPtr Marker() {
  return Object::unknown_constant().ptr();
}

DART_FORCE_INLINE
static bool CanShareObject(uword tags) {
  if ((tags & UntaggedObject::CanonicalBit::mask_in_place()) != 0) {
    return true;
  }
  const auto cid = UntaggedObject::ClassIdTag::decode(tags);
  if (cid == kOneByteStringCid) return true;
  if (cid == kTwoByteStringCid) return true;
  if (cid == kExternalOneByteStringCid) return true;
  if (cid == kExternalTwoByteStringCid) return true;
  if (cid == kMintCid) return true;
  if (cid == kImmutableArrayCid) return true;
  if (cid == kNeverCid) return true;
  if (cid == kSentinelCid) return true;
#if defined(DART_PRECOMPILED_RUNTIME)
  // In JIT mode we have field guards enabled which means
  // double/float32x4/float64x2 boxes can be mutable and we therefore cannot
  // share them.
  if (cid == kDoubleCid || cid == kFloat32x4Cid || cid == kFloat64x2Cid) {
    return true;
  }
#endif
  if (cid == kInt32x4Cid) return true;  // No field guards here.
  if (cid == kSendPortCid) return true;
  if (cid == kCapabilityCid) return true;
  if (cid == kRegExpCid) return true;

  return false;
}

// Whether executing `get:hashCode` (possibly in a different isolate) on an
// object with the given [tags] might return a different answer than the source
// object (if copying is needed) or on the same object (if the object is
// shared).
DART_FORCE_INLINE
static bool MightNeedReHashing(ObjectPtr object) {
  const uword tags = TagsFromUntaggedObject(object.untag());
  const auto cid = UntaggedObject::ClassIdTag::decode(tags);
  // These use structural hash codes and will therefore always result in the
  // same hash codes.
  if (cid == kOneByteStringCid) return false;
  if (cid == kTwoByteStringCid) return false;
  if (cid == kExternalOneByteStringCid) return false;
  if (cid == kExternalTwoByteStringCid) return false;
  if (cid == kMintCid) return false;
  if (cid == kDoubleCid) return false;
  if (cid == kSendPortCid) return false;
  if (cid == kCapabilityCid) return false;

  // These are shared and use identity hash codes. If they are used as a key in
  // a map or a value in a set, they will already have the identity hash code
  // set.
  if (cid == kImmutableArrayCid) return false;
  if (cid == kRegExpCid) return false;
  if (cid == kInt32x4Cid) return false;

  // We copy those (instead of sharing them) - see [CanShareObjct]. They rely
  // on the default hashCode implementation which uses identity hash codes
  // (instead of structural hash code).
  if (cid == kFloat32x4Cid || cid == kFloat64x2Cid) {
    return !kDartPrecompiledRuntime;
  }

  // If the [tags] indicates this is a canonical object we'll share it instead
  // of copying it. That would suggest we don't have to re-hash maps/sets
  // containing this object on the receiver side.
  //
  // Though the object can be a constant of a user-defined class with a
  // custom hash code that is misbehaving (e.g one that depends on global field
  // state, ...). To be on the safe side we'll force re-hashing if such objects
  // are encountered in maps/sets.
  //
  // => We might want to consider changing the implementation to avoid rehashing
  // in such cases in the future and disambiguate the documentation.
  return true;
}

DART_FORCE_INLINE
uword TagsFromUntaggedObject(UntaggedObject* obj) {
  return obj->tags_;
}

DART_FORCE_INLINE
void SetNewSpaceTaggingWord(ObjectPtr to, classid_t cid, uint32_t size) {
  uword tags = 0;

  tags = UntaggedObject::SizeTag::update(size, tags);
  tags = UntaggedObject::ClassIdTag::update(cid, tags);
  tags = UntaggedObject::OldBit::update(false, tags);
  tags = UntaggedObject::OldAndNotMarkedBit::update(false, tags);
  tags = UntaggedObject::OldAndNotRememberedBit::update(false, tags);
  tags = UntaggedObject::CanonicalBit::update(false, tags);
  tags = UntaggedObject::NewBit::update(true, tags);
#if defined(HASH_IN_OBJECT_HEADER)
  tags = UntaggedObject::HashTag::update(0, tags);
#endif
  to.untag()->tags_ = tags;
}

DART_FORCE_INLINE
ObjectPtr AllocateObject(intptr_t cid, intptr_t size) {
#if defined(DART_COMPRESSED_POINTERS)
  const bool compressed = true;
#else
  const bool compressed = false;
#endif
  return Object::Allocate(cid, size, Heap::kNew, compressed);
}

DART_FORCE_INLINE
void UpdateLengthField(intptr_t cid, ObjectPtr from, ObjectPtr to) {
  // We share these objects - never copy them.
  ASSERT(!IsStringClassId(cid));
  ASSERT(cid != kImmutableArrayCid);

  // We update any in-heap variable sized object with the length to keep the
  // length and the size in the object header in-sync for the GC.
  if (cid == kArrayCid) {
    static_cast<UntaggedArray*>(to.untag())->length_ =
        static_cast<UntaggedArray*>(from.untag())->length_;
  } else if (IsTypedDataClassId(cid)) {
    static_cast<UntaggedTypedDataBase*>(to.untag())->length_ =
        static_cast<UntaggedTypedDataBase*>(from.untag())->length_;
  }
}

void InitializeExternalTypedData(intptr_t cid,
                                 ExternalTypedDataPtr from,
                                 ExternalTypedDataPtr to) {
  auto raw_from = from.untag();
  auto raw_to = to.untag();
  const intptr_t length =
      TypedData::ElementSizeInBytes(cid) * Smi::Value(raw_from->length_);

  auto buffer = static_cast<uint8_t*>(malloc(length));
  memmove(buffer, raw_from->data_, length);
  raw_to->length_ = raw_from->length_;
  raw_to->data_ = buffer;
}

void InitializeTypedDataView(TypedDataViewPtr obj) {
  obj.untag()->typed_data_ = TypedDataBase::null();
  obj.untag()->offset_in_bytes_ = 0;
  obj.untag()->length_ = 0;
}

void FreeExternalTypedData(void* isolate_callback_data, void* buffer) {
  free(buffer);
}

void FreeTransferablePeer(void* isolate_callback_data, void* peer) {
  delete static_cast<TransferableTypedDataPeer*>(peer);
}

class ForwardMapBase {
 public:
  explicit ForwardMapBase(Thread* thread)
      : thread_(thread), zone_(thread->zone()), isolate_(thread->isolate()) {}

 protected:
  friend class ObjectGraphCopier;

  intptr_t GetObjectId(ObjectPtr object) {
    if (object->IsNewObject()) {
      return isolate_->forward_table_new()->GetValueExclusive(object);
    } else {
      return isolate_->forward_table_old()->GetValueExclusive(object);
    }
  }
  void SetObjectId(ObjectPtr object, intptr_t id) {
    if (object->IsNewObject()) {
      isolate_->forward_table_new()->SetValueExclusive(object, id);
    } else {
      isolate_->forward_table_old()->SetValueExclusive(object, id);
    }
  }

  void FinalizeTransferable(const TransferableTypedData& from,
                            const TransferableTypedData& to) {
    // Get the old peer.
    auto fpeer = static_cast<TransferableTypedDataPeer*>(
        thread_->heap()->GetPeer(from.ptr()));
    ASSERT(fpeer != nullptr && fpeer->data() != nullptr);
    const intptr_t length = fpeer->length();

    // Allocate new peer object with (data, length).
    auto tpeer = new TransferableTypedDataPeer(fpeer->data(), length);
    thread_->heap()->SetPeer(to.ptr(), tpeer);

    // Move the handle itself to the new object.
    fpeer->handle()->EnsureFreedExternal(thread_->isolate_group());
    tpeer->set_handle(FinalizablePersistentHandle::New(
        thread_->isolate_group(), to, tpeer, FreeTransferablePeer, length,
        /*auto_delete=*/true));
    fpeer->ClearData();
  }

  void FinalizeExternalTypedData(const ExternalTypedData& to) {
    to.AddFinalizer(to.DataAddr(0), &FreeExternalTypedData, to.LengthInBytes());
  }

  Thread* thread_;
  Zone* zone_;
  Isolate* isolate_;

 private:
  DISALLOW_COPY_AND_ASSIGN(ForwardMapBase);
};

class FastForwardMap : public ForwardMapBase {
 public:
  explicit FastForwardMap(Thread* thread)
      : ForwardMapBase(thread),
        raw_from_to_(thread->zone(), 20),
        raw_transferables_from_to_(thread->zone(), 0),
        raw_objects_to_rehash_(thread->zone(), 0) {
    raw_from_to_.Resize(2);
    raw_from_to_[0] = Object::null();
    raw_from_to_[1] = Object::null();
    fill_cursor_ = 2;
  }

  ObjectPtr ForwardedObject(ObjectPtr object) {
    const intptr_t id = GetObjectId(object);
    if (id == 0) return Marker();
    return raw_from_to_[id + 1];
  }

  void Insert(ObjectPtr from, ObjectPtr to) {
    ASSERT(ForwardedObject(from) == Marker());
    ASSERT(raw_from_to_.length() == raw_from_to_.length());
    const auto id = raw_from_to_.length();
    SetObjectId(from, id);
    raw_from_to_.Resize(id + 2);
    raw_from_to_[id] = from;
    raw_from_to_[id + 1] = to;
  }

  void AddTransferable(TransferableTypedDataPtr from,
                       TransferableTypedDataPtr to) {
    raw_transferables_from_to_.Add(from);
    raw_transferables_from_to_.Add(to);
  }
  void AddExternalTypedData(ExternalTypedDataPtr to) {
    raw_external_typed_data_to_.Add(to);
  }

  void AddObjectToRehash(ObjectPtr to) { raw_objects_to_rehash_.Add(to); }

 private:
  friend class FastObjectCopy;
  friend class ObjectGraphCopier;

  GrowableArray<ObjectPtr> raw_from_to_;
  GrowableArray<TransferableTypedDataPtr> raw_transferables_from_to_;
  GrowableArray<ExternalTypedDataPtr> raw_external_typed_data_to_;
  GrowableArray<ObjectPtr> raw_objects_to_rehash_;
  intptr_t fill_cursor_ = 0;

  DISALLOW_COPY_AND_ASSIGN(FastForwardMap);
};

class SlowForwardMap : public ForwardMapBase {
 public:
  explicit SlowForwardMap(Thread* thread)
      : ForwardMapBase(thread),
        from_to_(thread->zone(), 20),
        transferables_from_to_(thread->zone(), 0) {
    from_to_.Resize(2);
    from_to_[0] = &Object::null_object();
    from_to_[1] = &Object::null_object();
    fill_cursor_ = 2;
  }

  ObjectPtr ForwardedObject(ObjectPtr object) {
    const intptr_t id = GetObjectId(object);
    if (id == 0) return Marker();
    return from_to_[id + 1]->ptr();
  }

  void Insert(ObjectPtr from, ObjectPtr to) {
    ASSERT(ForwardedObject(from) == Marker());
    const auto id = from_to_.length();
    SetObjectId(from, id);
    from_to_.Resize(id + 2);
    from_to_[id] = &Object::Handle(Z, from);
    from_to_[id + 1] = &Object::Handle(Z, to);
  }

  void AddTransferable(const TransferableTypedData& from,
                       const TransferableTypedData& to) {
    transferables_from_to_.Add(&TransferableTypedData::Handle(from.ptr()));
    transferables_from_to_.Add(&TransferableTypedData::Handle(to.ptr()));
  }

  void AddExternalTypedData(ExternalTypedDataPtr to) {
    external_typed_data_.Add(&ExternalTypedData::Handle(to));
  }

  void AddObjectToRehash(const Object& to) {
    objects_to_rehash_.Add(&Object::Handle(to.ptr()));
  }

  void FinalizeTransferables() {
    for (intptr_t i = 0; i < transferables_from_to_.length(); i += 2) {
      auto from = transferables_from_to_[i];
      auto to = transferables_from_to_[i + 1];
      FinalizeTransferable(*from, *to);
    }
  }

  void FinalizeExternalTypedData() {
    for (intptr_t i = 0; i < external_typed_data_.length(); i++) {
      auto to = external_typed_data_[i];
      ForwardMapBase::FinalizeExternalTypedData(*to);
    }
  }

 private:
  friend class SlowObjectCopy;
  friend class ObjectGraphCopier;

  GrowableArray<const Object*> from_to_;
  GrowableArray<const TransferableTypedData*> transferables_from_to_;
  GrowableArray<const ExternalTypedData*> external_typed_data_;
  GrowableArray<const Object*> objects_to_rehash_;
  intptr_t fill_cursor_ = 0;

  DISALLOW_COPY_AND_ASSIGN(SlowForwardMap);
};

class ObjectCopyBase {
 public:
  explicit ObjectCopyBase(Thread* thread)
      : thread_(thread),
        heap_base_(thread->heap_base()),
        zone_(thread->zone()),
        heap_(thread->isolate_group()->heap()),
        class_table_(thread->isolate_group()->class_table()),
        new_space_(heap_->new_space()),
        tmp_(Object::Handle(thread->zone())) {}
  ~ObjectCopyBase() {}

 protected:
  static ObjectPtr LoadPointer(ObjectPtr src, intptr_t offset) {
    return src.untag()->LoadPointer(reinterpret_cast<ObjectPtr*>(
        reinterpret_cast<uint8_t*>(src.untag()) + offset));
  }
  static CompressedObjectPtr LoadCompressedPointer(ObjectPtr src,
                                                   intptr_t offset) {
    return src.untag()->LoadPointer(reinterpret_cast<CompressedObjectPtr*>(
        reinterpret_cast<uint8_t*>(src.untag()) + offset));
  }
  static compressed_uword LoadCompressedNonPointerWord(ObjectPtr src,
                                                       intptr_t offset) {
    return *reinterpret_cast<compressed_uword*>(
        reinterpret_cast<uint8_t*>(src.untag()) + offset);
  }
  static void StorePointerBarrier(ObjectPtr obj,
                                  intptr_t offset,
                                  ObjectPtr value) {
    obj.untag()->StorePointer(
        reinterpret_cast<ObjectPtr*>(reinterpret_cast<uint8_t*>(obj.untag()) +
                                     offset),
        value);
  }
  static void StoreCompressedPointerBarrier(ObjectPtr obj,
                                            intptr_t offset,
                                            ObjectPtr value) {
    obj.untag()->StoreCompressedPointer(
        reinterpret_cast<CompressedObjectPtr*>(
            reinterpret_cast<uint8_t*>(obj.untag()) + offset),
        value);
  }
  void StoreCompressedLargeArrayPointerBarrier(ObjectPtr obj,
                                               intptr_t offset,
                                               ObjectPtr value) {
    obj.untag()->StoreCompressedArrayPointer(
        reinterpret_cast<CompressedObjectPtr*>(
            reinterpret_cast<uint8_t*>(obj.untag()) + offset),
        value, thread_);
  }
  static void StorePointerNoBarrier(ObjectPtr obj,
                                    intptr_t offset,
                                    ObjectPtr value) {
    *reinterpret_cast<ObjectPtr*>(reinterpret_cast<uint8_t*>(obj.untag()) +
                                  offset) = value;
  }
  template <typename T = ObjectPtr>
  static void StoreCompressedPointerNoBarrier(ObjectPtr obj,
                                              intptr_t offset,
                                              T value) {
    *reinterpret_cast<CompressedObjectPtr*>(
        reinterpret_cast<uint8_t*>(obj.untag()) + offset) = value;
  }
  static void StoreCompressedNonPointerWord(ObjectPtr obj,
                                            intptr_t offset,
                                            compressed_uword value) {
    *reinterpret_cast<compressed_uword*>(
        reinterpret_cast<uint8_t*>(obj.untag()) + offset) = value;
  }

  DART_FORCE_INLINE
  bool CanCopyObject(uword tags, ObjectPtr object) {
    const auto cid = UntaggedObject::ClassIdTag::decode(tags);
    if (cid > kNumPredefinedCids) {
      const bool has_native_fields =
          Class::NumNativeFieldsOf(class_table_->At(cid)) != 0;
      if (has_native_fields) {
        exception_msg_ =
            "Illegal argument in isolate message: (object has native fields)";
        return false;
      }
      return true;
    }
#define HANDLE_ILLEGAL_CASE(Type)                                              \
  case k##Type##Cid: {                                                         \
    exception_msg_ =                                                           \
        "Illegal argument in isolate message: "                                \
        "(object is a" #Type ")";                                              \
    return false;                                                              \
  }

    switch (cid) {
      HANDLE_ILLEGAL_CASE(MirrorReference)
      HANDLE_ILLEGAL_CASE(ReceivePort)
      HANDLE_ILLEGAL_CASE(StackTrace)
      HANDLE_ILLEGAL_CASE(UserTag)
      HANDLE_ILLEGAL_CASE(DynamicLibrary)
      HANDLE_ILLEGAL_CASE(Pointer)
      case kClosureCid: {
        if (!Function::IsImplicitStaticClosureFunction(
                Closure::FunctionOf(Closure::RawCast(object)))) {
          exception_msg_ =
              "Illegal argument in isolate message: (object is a closure)";
          return false;
        }
        ASSERT(Closure::ContextOf(Closure::RawCast(object)) == Object::null());
        return true;
      }
      default:
        return true;
    }
  }

  Thread* thread_;
  uword heap_base_;
  Zone* zone_;
  Heap* heap_;
  ClassTable* class_table_;
  Scavenger* new_space_;
  Object& tmp_;

  const char* exception_msg_ = nullptr;
};

class FastObjectCopyBase : public ObjectCopyBase {
 public:
  using Types = PtrTypes;

  explicit FastObjectCopyBase(Thread* thread)
      : ObjectCopyBase(thread), fast_forward_map_(thread) {}

 protected:
  DART_FORCE_INLINE
  void ForwardCompressedPointers(ObjectPtr src,
                                 ObjectPtr dst,
                                 intptr_t offset,
                                 intptr_t end_offset) {
    for (; offset < end_offset; offset += kCompressedWordSize) {
      ForwardCompressedPointer(src, dst, offset);
    }
  }

  DART_FORCE_INLINE
  void ForwardCompressedPointers(ObjectPtr src,
                                 ObjectPtr dst,
                                 intptr_t offset,
                                 intptr_t end_offset,
                                 UnboxedFieldBitmap bitmap) {
    if (bitmap.IsEmpty()) {
      ForwardCompressedPointers(src, dst, offset, end_offset);
      return;
    }
    intptr_t bit = offset >> kCompressedWordSizeLog2;
    for (; offset < end_offset; offset += kCompressedWordSize) {
      if (bitmap.Get(bit++)) {
        StoreCompressedNonPointerWord(
            dst, offset, LoadCompressedNonPointerWord(src, offset));
      } else {
        ForwardCompressedPointer(src, dst, offset);
      }
    }
  }

  void ForwardCompressedArrayPointers(intptr_t array_length,
                                      ObjectPtr src,
                                      ObjectPtr dst,
                                      intptr_t offset,
                                      intptr_t end_offset) {
    for (; offset < end_offset; offset += kCompressedWordSize) {
      ForwardCompressedPointer(src, dst, offset);
    }
  }

  DART_FORCE_INLINE
  void ForwardCompressedPointer(ObjectPtr src, ObjectPtr dst, intptr_t offset) {
    auto value = LoadCompressedPointer(src, offset);
    if (!value.IsHeapObject()) {
      StoreCompressedPointerNoBarrier(dst, offset, value);
      return;
    }
    auto value_decompressed = value.Decompress(heap_base_);
    const uword tags = TagsFromUntaggedObject(value_decompressed.untag());
    if (CanShareObject(tags)) {
      StoreCompressedPointerNoBarrier(dst, offset, value);
      return;
    }

    ObjectPtr existing_to =
        fast_forward_map_.ForwardedObject(value_decompressed);
    if (existing_to != Marker()) {
      StoreCompressedPointerNoBarrier(dst, offset, existing_to);
      return;
    }

    if (UNLIKELY(!CanCopyObject(tags, value_decompressed))) {
      ASSERT(exception_msg_ != nullptr);
      StoreCompressedPointerNoBarrier(dst, offset, Object::null());
      return;
    }

    auto to = Forward(tags, value.Decompress(heap_base_));
    StoreCompressedPointerNoBarrier(dst, offset, to);
  }

  ObjectPtr Forward(uword tags, ObjectPtr from) {
    const intptr_t header_size = UntaggedObject::SizeTag::decode(tags);
    const auto cid = UntaggedObject::ClassIdTag::decode(tags);
    const uword size =
        header_size != 0 ? header_size : from.untag()->HeapSize();
    if (Heap::IsAllocatableInNewSpace(size)) {
      const uword alloc = new_space_->TryAllocate(thread_, size);
      if (alloc != 0) {
        ObjectPtr to(reinterpret_cast<UntaggedObject*>(alloc));
        fast_forward_map_.Insert(from, to);

        if (IsExternalTypedDataClassId(cid)) {
          SetNewSpaceTaggingWord(to, cid, header_size);
          InitializeExternalTypedData(cid, ExternalTypedData::RawCast(from),
                                      ExternalTypedData::RawCast(to));
          fast_forward_map_.AddExternalTypedData(
              ExternalTypedData::RawCast(to));
        } else if (IsTypedDataViewClassId(cid)) {
          // We set the views backing store to `null` to satisfy an assertion in
          // GCCompactor::VisitTypedDataViewPointers().
          SetNewSpaceTaggingWord(to, cid, header_size);
          InitializeTypedDataView(TypedDataView::RawCast(to));
        }
        return to;
      }
    }
    exception_msg_ = kFastAllocationFailed;
    return Marker();
  }

  void EnqueueTransferable(TransferableTypedDataPtr from,
                           TransferableTypedDataPtr to) {
    fast_forward_map_.AddTransferable(from, to);
  }
  void EnqueueObjectToRehash(ObjectPtr to) {
    fast_forward_map_.AddObjectToRehash(to);
  }

  static void StoreCompressedArrayPointers(intptr_t array_length,
                                           ObjectPtr src,
                                           ObjectPtr dst,
                                           intptr_t offset,
                                           intptr_t end_offset) {
    StoreCompressedPointers(src, dst, offset, end_offset);
  }
  static void StoreCompressedPointers(ObjectPtr src,
                                      ObjectPtr dst,
                                      intptr_t offset,
                                      intptr_t end_offset) {
    StoreCompressedPointersNoBarrier(src, dst, offset, end_offset);
  }
  static void StoreCompressedPointersNoBarrier(ObjectPtr src,
                                               ObjectPtr dst,
                                               intptr_t offset,
                                               intptr_t end_offset) {
    for (; offset <= end_offset; offset += kCompressedWordSize) {
      StoreCompressedPointerNoBarrier(dst, offset,
                                      LoadCompressedPointer(src, offset));
    }
  }

 protected:
  friend class ObjectGraphCopier;

  FastForwardMap fast_forward_map_;
};

class SlowObjectCopyBase : public ObjectCopyBase {
 public:
  using Types = HandleTypes;

  explicit SlowObjectCopyBase(Thread* thread)
      : ObjectCopyBase(thread), slow_forward_map_(thread) {}

 protected:
  DART_FORCE_INLINE
  void ForwardCompressedPointers(const Object& src,
                                 const Object& dst,
                                 intptr_t offset,
                                 intptr_t end_offset) {
    for (; offset < end_offset; offset += kCompressedWordSize) {
      ForwardCompressedPointer(src, dst, offset);
    }
  }

  DART_FORCE_INLINE
  void ForwardCompressedPointers(const Object& src,
                                 const Object& dst,
                                 intptr_t offset,
                                 intptr_t end_offset,
                                 UnboxedFieldBitmap bitmap) {
    intptr_t bit = offset >> kCompressedWordSizeLog2;
    for (; offset < end_offset; offset += kCompressedWordSize) {
      if (bitmap.Get(bit++)) {
        StoreCompressedNonPointerWord(
            dst.ptr(), offset, LoadCompressedNonPointerWord(src.ptr(), offset));
      } else {
        ForwardCompressedPointer(src, dst, offset);
      }
    }
  }

  void ForwardCompressedArrayPointers(intptr_t array_length,
                                      const Object& src,
                                      const Object& dst,
                                      intptr_t offset,
                                      intptr_t end_offset) {
    if (Array::UseCardMarkingForAllocation(array_length)) {
      for (; offset < end_offset; offset += kCompressedWordSize) {
        ForwardCompressedLargeArrayPointer(src, dst, offset);
      }
    } else {
      for (; offset < end_offset; offset += kCompressedWordSize) {
        ForwardCompressedPointer(src, dst, offset);
      }
    }
  }

  DART_FORCE_INLINE
  void ForwardCompressedLargeArrayPointer(const Object& src,
                                          const Object& dst,
                                          intptr_t offset) {
    auto value = LoadCompressedPointer(src.ptr(), offset);
    if (!value.IsHeapObject()) {
      StoreCompressedPointerNoBarrier(dst.ptr(), offset, value);
      return;
    }

    auto value_decompressed = value.Decompress(heap_base_);
    const uword tags = TagsFromUntaggedObject(value_decompressed.untag());
    if (CanShareObject(tags)) {
      StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset,
                                              value_decompressed);
      return;
    }

    ObjectPtr existing_to =
        slow_forward_map_.ForwardedObject(value_decompressed);
    if (existing_to != Marker()) {
      StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset, existing_to);
      return;
    }

    if (UNLIKELY(!CanCopyObject(tags, value_decompressed))) {
      ASSERT(exception_msg_ != nullptr);
      StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset,
                                              Object::null());
      return;
    }

    tmp_ = value_decompressed;
    tmp_ = Forward(tags, tmp_);  // Only this can cause allocation.
    StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset, tmp_.ptr());
  }
  DART_FORCE_INLINE
  void ForwardCompressedPointer(const Object& src,
                                const Object& dst,
                                intptr_t offset) {
    auto value = LoadCompressedPointer(src.ptr(), offset);
    if (!value.IsHeapObject()) {
      StoreCompressedPointerNoBarrier(dst.ptr(), offset, value);
      return;
    }
    auto value_decompressed = value.Decompress(heap_base_);
    const uword tags = TagsFromUntaggedObject(value_decompressed.untag());
    if (CanShareObject(tags)) {
      StoreCompressedPointerBarrier(dst.ptr(), offset, value_decompressed);
      return;
    }

    ObjectPtr existing_to =
        slow_forward_map_.ForwardedObject(value_decompressed);
    if (existing_to != Marker()) {
      StoreCompressedPointerBarrier(dst.ptr(), offset, existing_to);
      return;
    }

    if (UNLIKELY(!CanCopyObject(tags, value_decompressed))) {
      ASSERT(exception_msg_ != nullptr);
      StoreCompressedPointerNoBarrier(dst.ptr(), offset, Object::null());
      return;
    }

    tmp_ = value_decompressed;
    tmp_ = Forward(tags, tmp_);  // Only this can cause allocation.
    StoreCompressedPointerBarrier(dst.ptr(), offset, tmp_.ptr());
  }
  ObjectPtr Forward(uword tags, const Object& from) {
    const intptr_t cid = UntaggedObject::ClassIdTag::decode(tags);
    intptr_t size = UntaggedObject::SizeTag::decode(tags);
    if (size == 0) {
      size = from.ptr().untag()->HeapSize();
    }
    ObjectPtr to = AllocateObject(cid, size);
    slow_forward_map_.Insert(from.ptr(), to);
    UpdateLengthField(cid, from.ptr(), to);
    if (cid == kArrayCid && !Heap::IsAllocatableInNewSpace(size)) {
      to.untag()->SetCardRememberedBitUnsynchronized();
    }
    if (IsExternalTypedDataClassId(cid)) {
      InitializeExternalTypedData(cid, ExternalTypedData::RawCast(from.ptr()),
                                  ExternalTypedData::RawCast(to));
      slow_forward_map_.AddExternalTypedData(ExternalTypedData::RawCast(to));
    } else if (IsTypedDataViewClassId(cid)) {
      // We set the views backing store to `null` to satisfy an assertion in
      // GCCompactor::VisitTypedDataViewPointers().
      InitializeTypedDataView(TypedDataView::RawCast(to));
    }
    return to;
  }
  void EnqueueTransferable(const TransferableTypedData& from,
                           const TransferableTypedData& to) {
    slow_forward_map_.AddTransferable(from, to);
  }
  void EnqueueObjectToRehash(const Object& to) {
    slow_forward_map_.AddObjectToRehash(to);
  }

  void StoreCompressedArrayPointers(intptr_t array_length,
                                    const Object& src,
                                    const Object& dst,
                                    intptr_t offset,
                                    intptr_t end_offset) {
    auto src_ptr = src.ptr();
    auto dst_ptr = dst.ptr();
    if (Array::UseCardMarkingForAllocation(array_length)) {
      for (; offset <= end_offset; offset += kCompressedWordSize) {
        StoreCompressedLargeArrayPointerBarrier(
            dst_ptr, offset,
            LoadCompressedPointer(src_ptr, offset).Decompress(heap_base_));
      }
    } else {
      for (; offset <= end_offset; offset += kCompressedWordSize) {
        StoreCompressedPointerBarrier(
            dst_ptr, offset,
            LoadCompressedPointer(src_ptr, offset).Decompress(heap_base_));
      }
    }
  }
  void StoreCompressedPointers(const Object& src,
                               const Object& dst,
                               intptr_t offset,
                               intptr_t end_offset) {
    auto src_ptr = src.ptr();
    auto dst_ptr = dst.ptr();
    for (; offset <= end_offset; offset += kCompressedWordSize) {
      StoreCompressedPointerBarrier(
          dst_ptr, offset,
          LoadCompressedPointer(src_ptr, offset).Decompress(heap_base_));
    }
  }
  static void StoreCompressedPointersNoBarrier(const Object& src,
                                               const Object& dst,
                                               intptr_t offset,
                                               intptr_t end_offset) {
    auto src_ptr = src.ptr();
    auto dst_ptr = dst.ptr();
    for (; offset <= end_offset; offset += kCompressedWordSize) {
      StoreCompressedPointerNoBarrier(dst_ptr, offset,
                                      LoadCompressedPointer(src_ptr, offset));
    }
  }

 protected:
  friend class ObjectGraphCopier;

  SlowForwardMap slow_forward_map_;
};

template <typename Base>
class ObjectCopy : public Base {
 public:
  using Types = typename Base::Types;

  explicit ObjectCopy(Thread* thread) : Base(thread) {}

  void CopyPredefinedInstance(typename Types::Object from,
                              typename Types::Object to,
                              intptr_t cid) {
    if (IsImplicitFieldClassId(cid)) {
      CopyUserdefinedInstance(from, to);
      return;
    }
    switch (cid) {
#define COPY_TO(clazz)                                                         \
  case clazz::kClassId: {                                                      \
    typename Types::clazz casted_from = Types::Cast##clazz(from);              \
    typename Types::clazz casted_to = Types::Cast##clazz(to);                  \
    Copy##clazz(casted_from, casted_to);                                       \
    return;                                                                    \
  }

      CLASS_LIST_NO_OBJECT_NOR_STRING_NOR_ARRAY_NOR_MAP(COPY_TO)
      COPY_TO(Array)
      COPY_TO(LinkedHashMap)
      COPY_TO(LinkedHashSet)
#undef COPY_TO

#define COPY_TO(clazz) case kTypedData##clazz##Cid:

      CLASS_LIST_TYPED_DATA(COPY_TO) {
        typename Types::TypedData casted_from = Types::CastTypedData(from);
        typename Types::TypedData casted_to = Types::CastTypedData(to);
        CopyTypedData(casted_from, casted_to);
        return;
      }
#undef COPY_TO

      case kByteDataViewCid:
#define COPY_TO(clazz) case kTypedData##clazz##ViewCid:
        CLASS_LIST_TYPED_DATA(COPY_TO) {
          typename Types::TypedDataView casted_from =
              Types::CastTypedDataView(from);
          typename Types::TypedDataView casted_to =
              Types::CastTypedDataView(to);
          CopyTypedDataView(casted_from, casted_to);
          return;
        }
#undef COPY_TO

#define COPY_TO(clazz) case kExternalTypedData##clazz##Cid:

        CLASS_LIST_TYPED_DATA(COPY_TO) {
          typename Types::ExternalTypedData casted_from =
              Types::CastExternalTypedData(from);
          typename Types::ExternalTypedData casted_to =
              Types::CastExternalTypedData(to);
          CopyExternalTypedData(casted_from, casted_to);
          return;
        }
#undef COPY_TO
      default:
        break;
    }

    const Object& obj = Types::HandlifyObject(from);
    FATAL1("Unexpected object: %s\n", obj.ToCString());
  }

#if defined(DART_PRECOMPILED_RUNTIME)
  void CopyUserdefinedInstanceAOT(typename Types::Object from,
                                  typename Types::Object to,
                                  UnboxedFieldBitmap bitmap) {
    const intptr_t instance_size = UntagObject(from)->HeapSize();
    Base::ForwardCompressedPointers(from, to, kWordSize, instance_size, bitmap);
  }
#endif

  void CopyUserdefinedInstance(typename Types::Object from,
                               typename Types::Object to) {
    const intptr_t instance_size = UntagObject(from)->HeapSize();
    Base::ForwardCompressedPointers(from, to, kWordSize, instance_size);
  }

  void CopyClosure(typename Types::Closure from, typename Types::Closure to) {
    Base::StoreCompressedPointers(
        from, to, OFFSET_OF(UntaggedClosure, instantiator_type_arguments_),
        OFFSET_OF(UntaggedClosure, function_));
    Base::ForwardCompressedPointer(from, to,
                                   OFFSET_OF(UntaggedClosure, context_));
    Base::StoreCompressedPointersNoBarrier(from, to,
                                           OFFSET_OF(UntaggedClosure, hash_),
                                           OFFSET_OF(UntaggedClosure, hash_));
    ONLY_IN_PRECOMPILED(UntagClosure(to)->entry_point_ =
                            UntagClosure(from)->entry_point_);
  }

  void CopyArray(typename Types::Array from, typename Types::Array to) {
    const intptr_t length = Smi::Value(UntagArray(from)->length());
    Base::StoreCompressedArrayPointers(
        length, from, to, OFFSET_OF(UntaggedArray, type_arguments_),
        OFFSET_OF(UntaggedArray, type_arguments_));
    Base::StoreCompressedPointersNoBarrier(from, to,
                                           OFFSET_OF(UntaggedArray, length_),
                                           OFFSET_OF(UntaggedArray, length_));
    Base::ForwardCompressedArrayPointers(
        length, from, to, Array::data_offset(),
        Array::data_offset() + kCompressedWordSize * length);
  }

  void CopyGrowableObjectArray(typename Types::GrowableObjectArray from,
                               typename Types::GrowableObjectArray to) {
    Base::StoreCompressedPointers(
        from, to, OFFSET_OF(UntaggedGrowableObjectArray, type_arguments_),
        OFFSET_OF(UntaggedGrowableObjectArray, type_arguments_));
    Base::StoreCompressedPointersNoBarrier(
        from, to, OFFSET_OF(UntaggedGrowableObjectArray, length_),
        OFFSET_OF(UntaggedGrowableObjectArray, length_));
    Base::ForwardCompressedPointer(
        from, to, OFFSET_OF(UntaggedGrowableObjectArray, data_));
  }

  template <intptr_t one_for_set_two_for_map, typename T>
  void CopyLinkedHashBase(T from,
                          T to,
                          UntaggedLinkedHashBase* from_untagged,
                          UntaggedLinkedHashBase* to_untagged) {
    // We have to find out whether the map needs re-hashing on the receiver side
    // due to keys being copied and the keys therefore possibly having different
    // hash codes (e.g. due to user-defined hashCode implementation or due to
    // new identity hash codes of the copied objects).
    bool needs_rehashing = false;
    ArrayPtr data = from_untagged->data_.Decompress(Base::heap_base_);
    if (data != Array::null()) {
      UntaggedArray* untagged_data = data.untag();
      const intptr_t length = Smi::Value(untagged_data->length_);
      auto key_value_pairs = untagged_data->data();
      for (intptr_t i = 0; i < length; i += one_for_set_two_for_map) {
        ObjectPtr key = key_value_pairs[i].Decompress(Base::heap_base_);
        if (key->IsHeapObject()) {
          if (MightNeedReHashing(key)) {
            needs_rehashing = true;
            break;
          }
        }
      }
    }

    Base::StoreCompressedPointers(
        from, to, OFFSET_OF(UntaggedLinkedHashBase, type_arguments_),
        OFFSET_OF(UntaggedLinkedHashBase, type_arguments_));

    // Compared with the snapshot-based (de)serializer we do preserve the same
    // backing store (i.e. used_data/deleted_keys/data) and therefore do not
    // magically shrink backing store based on usage.
    //
    // We do this to avoid making assumptions about the object graph and the
    // linked hash map (e.g. assuming there's no other references to the data,
    // assuming the linked hashmap is in a consistent state)
    if (needs_rehashing) {
      to_untagged->hash_mask_ = Smi::New(0);
      to_untagged->index_ = TypedData::RawCast(Object::null());
      Base::EnqueueObjectToRehash(to);
    }

    // From this point on we shouldn't use the raw pointers, since GC might
    // happen when forwarding objects.
    from_untagged = nullptr;
    to_untagged = nullptr;

    if (!needs_rehashing) {
      Base::ForwardCompressedPointer(from, to,
                                     OFFSET_OF(UntaggedLinkedHashBase, index_));
      Base::StoreCompressedPointersNoBarrier(
          from, to, OFFSET_OF(UntaggedLinkedHashBase, hash_mask_),
          OFFSET_OF(UntaggedLinkedHashBase, hash_mask_));
    }
    Base::ForwardCompressedPointer(from, to,
                                   OFFSET_OF(UntaggedLinkedHashBase, data_));
    Base::StoreCompressedPointersNoBarrier(
        from, to, OFFSET_OF(UntaggedLinkedHashBase, used_data_),
        OFFSET_OF(UntaggedLinkedHashBase, used_data_));
    Base::StoreCompressedPointersNoBarrier(
        from, to, OFFSET_OF(UntaggedLinkedHashMap, deleted_keys_),
        OFFSET_OF(UntaggedLinkedHashMap, deleted_keys_));
  }

  void CopyLinkedHashMap(typename Types::LinkedHashMap from,
                         typename Types::LinkedHashMap to) {
    CopyLinkedHashBase<2, typename Types::LinkedHashMap>(
        from, to, UntagLinkedHashMap(from), UntagLinkedHashMap(to));
  }
  void CopyLinkedHashSet(typename Types::LinkedHashSet from,
                         typename Types::LinkedHashSet to) {
    CopyLinkedHashBase<1, typename Types::LinkedHashSet>(
        from, to, UntagLinkedHashSet(from), UntagLinkedHashSet(to));
  }

  void CopyDouble(typename Types::Double from, typename Types::Double to) {
#if !defined(DART_PRECOMPILED_RUNTIME)
    auto raw_from = UntagDouble(from);
    auto raw_to = UntagDouble(to);
    raw_to->value_ = raw_from->value_;
#else
    // Will be shared and not copied.
    UNREACHABLE();
#endif
  }

  void CopyFloat32x4(typename Types::Float32x4 from,
                     typename Types::Float32x4 to) {
#if !defined(DART_PRECOMPILED_RUNTIME)
    auto raw_from = UntagFloat32x4(from);
    auto raw_to = UntagFloat32x4(to);
    raw_to->value_[0] = raw_from->value_[0];
    raw_to->value_[1] = raw_from->value_[1];
    raw_to->value_[2] = raw_from->value_[2];
    raw_to->value_[3] = raw_from->value_[3];
#else
    // Will be shared and not copied.
    UNREACHABLE();
#endif
  }

  void CopyFloat64x2(typename Types::Float64x2 from,
                     typename Types::Float64x2 to) {
#if !defined(DART_PRECOMPILED_RUNTIME)
    auto raw_from = UntagFloat64x2(from);
    auto raw_to = UntagFloat64x2(to);
    raw_to->value_[0] = raw_from->value_[0];
    raw_to->value_[1] = raw_from->value_[1];
#else
    // Will be shared and not copied.
    UNREACHABLE();
#endif
  }

  void CopyTypedData(typename Types::TypedData from,
                     typename Types::TypedData to) {
    auto raw_from = UntagTypedData(from);
    auto raw_to = UntagTypedData(to);
    const intptr_t cid = Types::GetTypedDataPtr(from)->GetClassId();
    raw_to->length_ = raw_from->length_;
    raw_to->RecomputeDataField();
    const intptr_t length =
        TypedData::ElementSizeInBytes(cid) * Smi::Value(raw_from->length_);
    memmove(raw_to->data_, raw_from->data_, length);
  }

  void CopyTypedDataView(typename Types::TypedDataView from,
                         typename Types::TypedDataView to) {
    // This will forward & initialize the typed data.
    Base::ForwardCompressedPointer(
        from, to, OFFSET_OF(UntaggedTypedDataView, typed_data_));

    auto raw_from = UntagTypedDataView(from);
    auto raw_to = UntagTypedDataView(to);
    raw_to->length_ = raw_from->length_;
    raw_to->offset_in_bytes_ = raw_from->offset_in_bytes_;
    raw_to->data_ = nullptr;

    if (raw_to->typed_data_.Decompress(Base::heap_base_) == Object::null()) {
      ASSERT(Base::exception_msg_ != nullptr);
      return;
    }

    const bool is_external =
        raw_from->data_ != raw_from->DataFieldForInternalTypedData();
    if (is_external) {
      // The raw_to is fully initialized at this point (see handling of external
      // typed data in [ForwardCompressedPointer])
      raw_to->RecomputeDataField();
    } else {
      // The raw_to isn't initialized yet, but it's address is valid, so we can
      // compute the data field it would use.
      raw_to->RecomputeDataFieldForInternalTypedData();
    }
    const bool is_external2 =
        raw_to->data_ != raw_to->DataFieldForInternalTypedData();
    ASSERT(is_external == is_external2);
  }

  void CopyExternalTypedData(typename Types::ExternalTypedData from,
                             typename Types::ExternalTypedData to) {
    // The external typed data is initialized on the forwarding pass (where
    // normally allocation but not initialization happens), so views on it
    // can be initialized immediately.
#if defined(DEBUG)
    auto raw_from = UntagExternalTypedData(from);
    auto raw_to = UntagExternalTypedData(to);
    ASSERT(raw_to->data_ != nullptr);
    ASSERT(raw_to->length_ == raw_from->length_);
#endif
  }

  void CopyTransferableTypedData(typename Types::TransferableTypedData from,
                                 typename Types::TransferableTypedData to) {
    // The [TransferableTypedData] is an empty object with an associated heap
    // peer object.
    // -> We'll validate that there's a peer and enqueue the transferable to be
    // transferred if the transitive copy is successful.
    auto fpeer = static_cast<TransferableTypedDataPeer*>(
        Base::heap_->GetPeer(Types::GetTransferableTypedDataPtr(from)));
    ASSERT(fpeer != nullptr);
    if (fpeer->data() == nullptr) {
      Base::exception_msg_ =
          "Illegal argument in isolate message"
          " : (TransferableTypedData has been transferred already)";
      return;
    }
    Base::EnqueueTransferable(from, to);
  }

#define DEFINE_UNSUPPORTED(clazz)                                              \
  void Copy##clazz(typename Types::clazz from, typename Types::clazz to) {     \
    FATAL("Objects of type " #clazz " should not occur in object graphs");     \
  }

  FOR_UNSUPPORTED_CLASSES(DEFINE_UNSUPPORTED)

#undef DEFINE_UNSUPPORTED

  UntaggedObject* UntagObject(typename Types::Object obj) {
    return Types::GetObjectPtr(obj).Decompress(Base::heap_base_).untag();
  }

#define DO(V)                                                                  \
  DART_FORCE_INLINE                                                            \
  Untagged##V* Untag##V(typename Types::V obj) {                               \
    return Types::Get##V##Ptr(obj).Decompress(Base::heap_base_).untag();       \
  }
  CLASS_LIST_FOR_HANDLES(DO)
#undef DO
};

class FastObjectCopy : public ObjectCopy<FastObjectCopyBase> {
 public:
  explicit FastObjectCopy(Thread* thread) : ObjectCopy(thread) {}
  ~FastObjectCopy() {}

  ObjectPtr TryCopyGraphFast(ObjectPtr root) {
    NoSafepointScope no_safepoint_scope;

    ObjectPtr root_copy = Forward(TagsFromUntaggedObject(root.untag()), root);
    if (root_copy == Marker()) {
      return root_copy;
    }
    while (fast_forward_map_.fill_cursor_ <
           fast_forward_map_.raw_from_to_.length()) {
      const intptr_t index = fast_forward_map_.fill_cursor_;
      ObjectPtr from = fast_forward_map_.raw_from_to_[index];
      ObjectPtr to = fast_forward_map_.raw_from_to_[index + 1];
      FastCopyObject(from, to);
      if (exception_msg_ != nullptr) {
        return root_copy;
      }
      fast_forward_map_.fill_cursor_ += 2;
    }
    if (root_copy != Marker()) {
      TryBuildArrayOfObjectsToRehash();
    }
    return root_copy;
  }

  void TryBuildArrayOfObjectsToRehash() {
    const auto& objects_to_rehash = fast_forward_map_.raw_objects_to_rehash_;
    const intptr_t length = objects_to_rehash.length();
    if (length == 0) return;

    const intptr_t size = Array::InstanceSize(length);
    const uword array_addr = new_space_->TryAllocate(thread_, size);
    if (array_addr == 0) {
      exception_msg_ = kFastAllocationFailed;
      return;
    }

    const uword header_size =
        UntaggedObject::SizeTag::SizeFits(size) ? size : 0;
    ArrayPtr array(reinterpret_cast<UntaggedArray*>(array_addr));
    SetNewSpaceTaggingWord(array, kArrayCid, header_size);
    StoreCompressedPointerNoBarrier(array, OFFSET_OF(UntaggedArray, length_),
                                    Smi::New(length));
    StoreCompressedPointerNoBarrier(array,
                                    OFFSET_OF(UntaggedArray, type_arguments_),
                                    TypeArguments::null());
    auto array_data = array.untag()->data();
    for (intptr_t i = 0; i < length; ++i) {
      array_data[i] = objects_to_rehash[i];
    }
    raw_objects_to_rehash_ = array;
  }

 private:
  friend class ObjectGraphCopier;

  void FastCopyObject(ObjectPtr from, ObjectPtr to) {
    const uword tags = TagsFromUntaggedObject(from.untag());
    const intptr_t cid = UntaggedObject::ClassIdTag::decode(tags);
    const intptr_t size = UntaggedObject::SizeTag::decode(tags);

    // Ensure the last word is GC-safe (our heap objects are 2-word aligned, the
    // object header stores the size in multiples of kObjectAlignment, the GC
    // uses the information from the header and therefore might visit one slot
    // more than the actual size of the instance).
    *reinterpret_cast<ObjectPtr*>(UntaggedObject::ToAddr(to) +
                                  from.untag()->HeapSize() - kWordSize) = 0;
    SetNewSpaceTaggingWord(to, cid, size);

    // Fall back to virtual variant for predefined classes
    if (cid < kNumPredefinedCids && cid != kInstanceCid) {
      CopyPredefinedInstance(from, to, cid);
      return;
    }
#if defined(DART_PRECOMPILED_RUNTIME)
    const auto bitmap =
        class_table_->shared_class_table()->GetUnboxedFieldsMapAt(cid);
    CopyUserdefinedInstanceAOT(Instance::RawCast(from), Instance::RawCast(to),
                               bitmap);
#else
    CopyUserdefinedInstance(Instance::RawCast(from), Instance::RawCast(to));
#endif
  }

  ArrayPtr raw_objects_to_rehash_ = Array::null();
};

class SlowObjectCopy : public ObjectCopy<SlowObjectCopyBase> {
 public:
  explicit SlowObjectCopy(Thread* thread)
      : ObjectCopy(thread), objects_to_rehash_(Array::Handle(thread->zone())) {}
  ~SlowObjectCopy() {}

  ObjectPtr ContinueCopyGraphSlow(const Object& root,
                                  const Object& fast_root_copy) {
    auto& root_copy = Object::Handle(Z, fast_root_copy.ptr());
    if (root_copy.ptr() == Marker()) {
      root_copy = Forward(TagsFromUntaggedObject(root.ptr().untag()), root);
    }

    Object& from = Object::Handle(Z);
    Object& to = Object::Handle(Z);
    while (slow_forward_map_.fill_cursor_ <
           slow_forward_map_.from_to_.length()) {
      const intptr_t index = slow_forward_map_.fill_cursor_;
      from = slow_forward_map_.from_to_[index]->ptr();
      to = slow_forward_map_.from_to_[index + 1]->ptr();
      CopyObject(from, to);
      slow_forward_map_.fill_cursor_ += 2;
      if (exception_msg_ != nullptr) {
        return Marker();
      }
    }
    BuildArrayOfObjectsToRehash();
    return root_copy.ptr();
  }

  void BuildArrayOfObjectsToRehash() {
    const auto& objects_to_rehash = slow_forward_map_.objects_to_rehash_;
    const intptr_t length = objects_to_rehash.length();
    if (length == 0) return;

    objects_to_rehash_ = Array::New(length);
    for (intptr_t i = 0; i < length; ++i) {
      objects_to_rehash_.SetAt(i, *objects_to_rehash[i]);
    }
  }

 private:
  friend class ObjectGraphCopier;

  void CopyObject(const Object& from, const Object& to) {
    const auto cid = from.GetClassId();

    // Fall back to virtual variant for predefined classes
    if (cid < kNumPredefinedCids && cid != kInstanceCid) {
      CopyPredefinedInstance(from, to, cid);
      return;
    }
#if defined(DART_PRECOMPILED_RUNTIME)
    const auto bitmap =
        class_table_->shared_class_table()->GetUnboxedFieldsMapAt(cid);
    CopyUserdefinedInstanceAOT(from, to, bitmap);
#else
    CopyUserdefinedInstance(from, to);
#endif
  }

  Array& objects_to_rehash_;
};

class ObjectGraphCopier {
 public:
  explicit ObjectGraphCopier(Thread* thread)
      : thread_(thread),
        zone_(thread->zone()),
        fast_object_copy_(thread_),
        slow_object_copy_(thread_) {
    thread_->isolate()->set_forward_table_new(new WeakTable());
    thread_->isolate()->set_forward_table_old(new WeakTable());
  }
  ~ObjectGraphCopier() {
    thread_->isolate()->set_forward_table_new(nullptr);
    thread_->isolate()->set_forward_table_old(nullptr);
  }

  // Result will be [<msg>, <objects-in-msg-to-rehash>]
  ObjectPtr CopyObjectGraph(const Object& root) {
    const char* exception_msg = nullptr;
    auto& result = Object::Handle(zone_);

    {
      LongJumpScope jump;  // e.g. for OOMs.
      if (setjmp(*jump.Set()) == 0) {
        result = CopyObjectGraphInternal(root, &exception_msg);
        // Any allocated external typed data must have finalizers attached so
        // memory will get free()ed.
        slow_object_copy_.slow_forward_map_.FinalizeExternalTypedData();
      } else {
        // Any allocated external typed data must have finalizers attached so
        // memory will get free()ed.
        slow_object_copy_.slow_forward_map_.FinalizeExternalTypedData();

        // The copy failed due to non-application error (e.g. OOM error),
        // propagate this error.
        result = thread_->StealStickyError();
        RELEASE_ASSERT(result.IsError());
      }
    }

    if (result.IsError()) {
      Exceptions::PropagateError(Error::Cast(result));
      UNREACHABLE();
    }
    if (result.ptr() == Marker()) {
      ASSERT(exception_msg != nullptr);
      ThrowException(exception_msg);
      UNREACHABLE();
    }

    // The copy was successful, then detach transferable data from the sender
    // and attach to the copied graph.
    slow_object_copy_.slow_forward_map_.FinalizeTransferables();
    return result.ptr();
  }

 private:
  ObjectPtr CopyObjectGraphInternal(const Object& root,
                                    const char** exception_msg) {
    const auto& result_array = Array::Handle(zone_, Array::New(2));
    if (!root.ptr()->IsHeapObject()) {
      result_array.SetAt(0, root);
      return result_array.ptr();
    }
    const uword tags = TagsFromUntaggedObject(root.ptr().untag());
    if (CanShareObject(tags)) {
      result_array.SetAt(0, root);
      return result_array.ptr();
    }
    if (!fast_object_copy_.CanCopyObject(tags, root.ptr())) {
      ASSERT(fast_object_copy_.exception_msg_ != nullptr);
      *exception_msg = fast_object_copy_.exception_msg_;
      return Marker();
    }

    // We try a fast new-space only copy first that will not use any barriers.
    auto& result = Object::Handle(Z, Marker());

    // All allocated but non-initialized heap objects have to be made GC-visible
    // at this point.
    if (FLAG_enable_fast_object_copy) {
      {
        NoSafepointScope no_safepoint_scope;

        result = fast_object_copy_.TryCopyGraphFast(root.ptr());
        if (result.ptr() != Marker()) {
          if (fast_object_copy_.exception_msg_ == nullptr) {
            result_array.SetAt(0, result);
            fast_object_copy_.tmp_ = fast_object_copy_.raw_objects_to_rehash_;
            result_array.SetAt(1, fast_object_copy_.tmp_);
            HandlifyExternalTypedData();
            HandlifyTransferables();
            return result_array.ptr();
          }

          // There are left-over uninitialized objects we'll have to make GC
          // visible.
          SwitchToSlowFowardingList();
        }
      }

      if (FLAG_gc_on_foc_slow_path) {
        // We use kLowMemory to force the GC to compact, which is more likely to
        // discover untracked pointers (and other issues, like incorrect class
        // table).
        thread_->heap()->CollectAllGarbage(Heap::kLowMemory);
      }

      // Fast copy failed due to
      //   - either failure to allocate into new space
      //   - or failure to copy object which we cannot copy
      ASSERT(fast_object_copy_.exception_msg_ != nullptr);
      if (fast_object_copy_.exception_msg_ != kFastAllocationFailed) {
        *exception_msg = fast_object_copy_.exception_msg_;
        return Marker();
      }
      ASSERT(fast_object_copy_.exception_msg_ == kFastAllocationFailed);
    }

    // Use the slow copy approach.
    result = slow_object_copy_.ContinueCopyGraphSlow(root, result);
    ASSERT((result.ptr() == Marker()) ==
           (slow_object_copy_.exception_msg_ != nullptr));
    if (result.ptr() == Marker()) {
      *exception_msg = slow_object_copy_.exception_msg_;
      return Marker();
    }

    result_array.SetAt(0, result);
    result_array.SetAt(1, slow_object_copy_.objects_to_rehash_);
    return result_array.ptr();
  }

  void SwitchToSlowFowardingList() {
    auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
    auto& slow_forward_map = slow_object_copy_.slow_forward_map_;

    MakeUninitializedNewSpaceObjectsGCSafe();
    HandlifyTransferables();
    HandlifyExternalTypedData();
    HandlifyObjectsToReHash();
    HandlifyFromToObjects();
    slow_forward_map.fill_cursor_ = fast_forward_map.fill_cursor_;
  }

  void MakeUninitializedNewSpaceObjectsGCSafe() {
    auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
    const auto length = fast_forward_map.raw_from_to_.length();
    const auto cursor = fast_forward_map.fill_cursor_;
    for (intptr_t i = cursor; i < length; i += 2) {
      auto from = fast_forward_map.raw_from_to_[i];
      auto to = fast_forward_map.raw_from_to_[i + 1];
      const uword tags = TagsFromUntaggedObject(from.untag());
      const intptr_t cid = UntaggedObject::ClassIdTag::decode(tags);
      const intptr_t size = UntaggedObject::SizeTag::decode(tags);
      // External typed data is already initialized.
      if (!IsExternalTypedDataClassId(cid) && !IsTypedDataViewClassId(cid)) {
        memset(to.untag(), 0x0, from.untag()->HeapSize());
        SetNewSpaceTaggingWord(to, cid, size);
        UpdateLengthField(cid, from, to);
      }
    }
  }
  void HandlifyTransferables() {
    auto& raw_transferables =
        fast_object_copy_.fast_forward_map_.raw_transferables_from_to_;
    const auto length = raw_transferables.length();
    if (length > 0) {
      auto& transferables =
          slow_object_copy_.slow_forward_map_.transferables_from_to_;
      transferables.Resize(length);
      for (intptr_t i = 0; i < length; i++) {
        transferables[i] =
            &TransferableTypedData::Handle(Z, raw_transferables[i]);
      }
      raw_transferables.Clear();
    }
  }
  void HandlifyExternalTypedData() {
    auto& raw_external_typed_data =
        fast_object_copy_.fast_forward_map_.raw_external_typed_data_to_;
    const auto length = raw_external_typed_data.length();
    if (length > 0) {
      auto& external_typed_data =
          slow_object_copy_.slow_forward_map_.external_typed_data_;
      external_typed_data.Resize(length);
      for (intptr_t i = 0; i < length; i++) {
        external_typed_data[i] =
            &ExternalTypedData::Handle(Z, raw_external_typed_data[i]);
      }
      raw_external_typed_data.Clear();
    }
  }
  void HandlifyObjectsToReHash() {
    auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
    auto& slow_forward_map = slow_object_copy_.slow_forward_map_;
    const auto length = fast_forward_map.raw_transferables_from_to_.length();
    if (length > 0) {
      slow_forward_map.objects_to_rehash_.Resize(length);
      for (intptr_t i = 0; i < length; i++) {
        slow_forward_map.objects_to_rehash_[i] =
            &Object::Handle(Z, fast_forward_map.raw_objects_to_rehash_[i]);
      }
      fast_forward_map.raw_objects_to_rehash_.Clear();
    }
  }
  void HandlifyFromToObjects() {
    auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
    auto& slow_forward_map = slow_object_copy_.slow_forward_map_;

    const intptr_t cursor = fast_forward_map.fill_cursor_;
    const intptr_t length = fast_forward_map.raw_from_to_.length();

    slow_forward_map.from_to_.Resize(length);
    for (intptr_t i = 2; i < length; i += 2) {
      slow_forward_map.from_to_[i] =
          i < cursor ? nullptr
                     : &Object::Handle(Z, fast_forward_map.raw_from_to_[i]);
      slow_forward_map.from_to_[i + 1] =
          &Object::Handle(Z, fast_forward_map.raw_from_to_[i + 1]);
    }
    fast_forward_map.raw_from_to_.Clear();
  }
  void ThrowException(const char* exception_msg) {
    const auto& msg_obj = String::Handle(Z, String::New(exception_msg));
    const auto& args = Array::Handle(Z, Array::New(1));
    args.SetAt(0, msg_obj);
    Exceptions::ThrowByType(Exceptions::kArgument, args);
    UNREACHABLE();
  }

  Thread* thread_;
  Zone* zone_;
  FastObjectCopy fast_object_copy_;
  SlowObjectCopy slow_object_copy_;
};

ObjectPtr CopyMutableObjectGraph(const Object& object) {
  auto thread = Thread::Current();
  ObjectGraphCopier copier(thread);
  return copier.CopyObjectGraph(object);
}

}  // namespace dart