Google Git
Sign in
dart / sdk.git / 2b5d2621df9a3911aa6aaad1830802b943c5059c / . / runtime / vm / object_graph_copy.cc
blob: d9fdaac5a5c5378dce72146bc89d40b17aaddccc [file] [log] [blame]
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/object_graph_copy.h"
#include <memory>
#include "vm/dart_api_state.h"
#include "vm/flags.h"
#include "vm/heap/weak_table.h"
#include "vm/longjump.h"
#include "vm/object.h"
#include "vm/object_store.h"
#include "vm/snapshot.h"
#include "vm/symbols.h"
#include "vm/timeline.h"
#define Z zone_
// The list here contains two kinds of classes of objects
// * objects that will be shared and we will therefore never need to copy
// * objects that user object graphs should never reference
#define FOR_UNSUPPORTED_CLASSES(V) \
V(AbstractType) \
V(ApiError) \
V(Bool) \
V(CallSiteData) \
V(Capability) \
V(Class) \
V(ClosureData) \
V(Code) \
V(CodeSourceMap) \
V(CompressedStackMaps) \
V(ContextScope) \
V(Bytecode) \
V(DynamicLibrary) \
V(Error) \
V(ExceptionHandlers) \
V(FfiTrampolineData) \
V(Field) \
V(Finalizer) \
V(FinalizerBase) \
V(FinalizerEntry) \
V(NativeFinalizer) \
V(Function) \
V(FunctionType) \
V(FutureOr) \
V(ICData) \
V(Instance) \
V(Instructions) \
V(InstructionsSection) \
V(InstructionsTable) \
V(Int32x4) \
V(Integer) \
V(KernelProgramInfo) \
V(LanguageError) \
V(Library) \
V(LibraryPrefix) \
V(LoadingUnit) \
V(LocalVarDescriptors) \
V(MegamorphicCache) \
V(Mint) \
V(MirrorReference) \
V(MonomorphicSmiableCall) \
V(Namespace) \
V(Number) \
V(ObjectPool) \
V(PatchClass) \
V(PcDescriptors) \
V(Pointer) \
V(ReceivePort) \
V(RecordType) \
V(RegExp) \
V(Script) \
V(Sentinel) \
V(SendPort) \
V(SingleTargetCache) \
V(Smi) \
V(StackTrace) \
V(SubtypeTestCache) \
V(SuspendState) \
V(Type) \
V(TypeArguments) \
V(TypeParameter) \
V(TypeParameters) \
V(TypedDataBase) \
V(UnhandledException) \
V(UnlinkedCall) \
V(UnwindError) \
V(UserTag) \
V(WeakArray) \
V(WeakSerializationReference)
namespace dart {
DEFINE_FLAG(bool,
enable_fast_object_copy,
true,
"Enable fast path for fast object copy.");
DEFINE_FLAG(bool,
gc_on_foc_slow_path,
false,
"Cause a GC when falling off the fast path for fast object copy.");
const char* kFastAllocationFailed = "fast allocation failed";
struct PtrTypes {
using Object = ObjectPtr;
static const dart::UntaggedObject* UntagObject(Object arg) {
return arg.untag();
}
static const dart::ObjectPtr GetObjectPtr(Object arg) { return arg; }
static const dart::Object& HandlifyObject(ObjectPtr arg) {
return dart::Object::Handle(arg);
}
#define DO(V) \
using V = V##Ptr; \
static Untagged##V* Untag##V(V##Ptr arg) { return arg.untag(); } \
static V##Ptr Get##V##Ptr(V##Ptr arg) { return arg; } \
static V##Ptr Cast##V(ObjectPtr arg) { return dart::V::RawCast(arg); }
CLASS_LIST_FOR_HANDLES(DO)
#undef DO
};
struct HandleTypes {
using Object = const dart::Object&;
static const dart::UntaggedObject* UntagObject(Object arg) {
return arg.ptr().untag();
}
static dart::ObjectPtr GetObjectPtr(Object arg) { return arg.ptr(); }
static Object HandlifyObject(Object arg) { return arg; }
#define DO(V) \
using V = const dart::V&; \
static Untagged##V* Untag##V(V arg) { return arg.ptr().untag(); } \
static V##Ptr Get##V##Ptr(V arg) { return arg.ptr(); } \
static V Cast##V(const dart::Object& arg) { return dart::V::Cast(arg); }
CLASS_LIST_FOR_HANDLES(DO)
#undef DO
};
DART_FORCE_INLINE
static ObjectPtr Marker() {
return Object::unknown_constant().ptr();
}
DART_FORCE_INLINE
static bool CanShareObject(ObjectPtr obj, uword tags) {
if ((tags & UntaggedObject::CanonicalBit::mask_in_place()) != 0) {
return true;
}
const auto cid = UntaggedObject::ClassIdTag::decode(tags);
if ((tags & UntaggedObject::ImmutableBit::mask_in_place()) != 0) {
if (IsUnmodifiableTypedDataViewClassId(cid)) {
// Unmodifiable typed data views may have mutable backing stores.
return TypedDataView::RawCast(obj)
->untag()
->typed_data()
->untag()
->IsImmutable();
}
// All other objects that have immutability bit set are deeply immutable.
return true;
}
// TODO(https://dartbug.com/55136): Mark Closures as shallowly imutable.
// And move this into the if above.
if (cid == kClosureCid) {
// We can share a closure iff it doesn't close over any state.
return Closure::RawCast(obj)->untag()->context() == Object::null();
}
return false;
}
bool CanShareObjectAcrossIsolates(ObjectPtr obj) {
if (!obj->IsHeapObject()) return true;
const uword tags = TagsFromUntaggedObject(obj.untag());
return CanShareObject(obj, tags);
}
// Whether executing `get:hashCode` (possibly in a different isolate) on an
// object with the given [tags] might return a different answer than the source
// object (if copying is needed) or on the same object (if the object is
// shared).
DART_FORCE_INLINE
static bool MightNeedReHashing(ObjectPtr object) {
const uword tags = TagsFromUntaggedObject(object.untag());
const auto cid = UntaggedObject::ClassIdTag::decode(tags);
// These use structural hash codes and will therefore always result in the
// same hash codes.
if (cid == kOneByteStringCid) return false;
if (cid == kTwoByteStringCid) return false;
if (cid == kMintCid) return false;
if (cid == kDoubleCid) return false;
if (cid == kBoolCid) return false;
if (cid == kSendPortCid) return false;
if (cid == kCapabilityCid) return false;
if (cid == kNullCid) return false;
// These are shared and use identity hash codes. If they are used as a key in
// a map or a value in a set, they will already have the identity hash code
// set.
if (cid == kRegExpCid) return false;
if (cid == kInt32x4Cid) return false;
// If the [tags] indicates this is a canonical object we'll share it instead
// of copying it. That would suggest we don't have to re-hash maps/sets
// containing this object on the receiver side.
//
// Though the object can be a constant of a user-defined class with a
// custom hash code that is misbehaving (e.g one that depends on global field
// state, ...). To be on the safe side we'll force re-hashing if such objects
// are encountered in maps/sets.
//
// => We might want to consider changing the implementation to avoid rehashing
// in such cases in the future and disambiguate the documentation.
return true;
}
DART_FORCE_INLINE
uword TagsFromUntaggedObject(UntaggedObject* obj) {
return obj->tags_;
}
DART_FORCE_INLINE
void SetNewSpaceTaggingWord(ObjectPtr to, classid_t cid, uint32_t size) {
uword tags = 0;
tags = UntaggedObject::SizeTag::update(size, tags);
tags = UntaggedObject::ClassIdTag::update(cid, tags);
tags = UntaggedObject::AlwaysSetBit::update(true, tags);
tags = UntaggedObject::NotMarkedBit::update(true, tags);
tags = UntaggedObject::OldAndNotRememberedBit::update(false, tags);
tags = UntaggedObject::CanonicalBit::update(false, tags);
tags = UntaggedObject::NewOrEvacuationCandidateBit::update(true, tags);
tags = UntaggedObject::ImmutableBit::update(
IsUnmodifiableTypedDataViewClassId(cid), tags);
#if defined(HASH_IN_OBJECT_HEADER)
tags = UntaggedObject::HashTag::update(0, tags);
#endif
to.untag()->tags_ = tags;
}
DART_FORCE_INLINE
ObjectPtr AllocateObject(intptr_t cid,
intptr_t size,
intptr_t allocated_bytes) {
#if defined(DART_COMPRESSED_POINTERS)
const bool compressed = true;
#else
const bool compressed = false;
#endif
const intptr_t kLargeMessageThreshold = 16 * MB;
const Heap::Space space =
allocated_bytes > kLargeMessageThreshold ? Heap::kOld : Heap::kNew;
// Mimic the old initialization behavior of Object::InitializeObject where
// the contents are initialized to Object::null(), except for TypedDataBase
// subclasses which are initialized to 0, as the contents of the original
// are translated and copied over prior to returning the object graph root.
if (IsTypedDataBaseClassId(cid)) {
return Object::Allocate(cid, size, space, compressed,
Object::from_offset<TypedDataBase>(),
Object::to_offset<TypedDataBase>());
} else {
// Remember that ptr_field_end_offset is the offset to the last Ptr
// field, not the offset just past it.
const uword ptr_field_end_offset =
size - (compressed ? kCompressedWordSize : kWordSize);
return Object::Allocate(cid, size, space, compressed,
Object::from_offset<Object>(),
ptr_field_end_offset);
}
}
DART_FORCE_INLINE
void UpdateLengthField(intptr_t cid, ObjectPtr from, ObjectPtr to) {
// We share these objects - never copy them.
ASSERT(!IsStringClassId(cid));
// We update any in-heap variable sized object with the length to keep the
// length and the size in the object header in-sync for the GC.
if (cid == kArrayCid || cid == kImmutableArrayCid) {
static_cast<UntaggedArray*>(to.untag())->length_ =
static_cast<UntaggedArray*>(from.untag())->length_;
} else if (cid == kContextCid) {
static_cast<UntaggedContext*>(to.untag())->num_variables_ =
static_cast<UntaggedContext*>(from.untag())->num_variables_;
} else if (IsTypedDataClassId(cid)) {
static_cast<UntaggedTypedDataBase*>(to.untag())->length_ =
static_cast<UntaggedTypedDataBase*>(from.untag())->length_;
} else if (cid == kRecordCid) {
static_cast<UntaggedRecord*>(to.untag())->shape_ =
static_cast<UntaggedRecord*>(from.untag())->shape_;
}
}
void InitializeExternalTypedData(intptr_t cid,
ExternalTypedDataPtr from,
ExternalTypedDataPtr to) {
auto raw_from = from.untag();
auto raw_to = to.untag();
const intptr_t length =
TypedData::ElementSizeInBytes(cid) * Smi::Value(raw_from->length_);
auto buffer = static_cast<uint8_t*>(malloc(length));
memmove(buffer, raw_from->data_, length);
raw_to->length_ = raw_from->length_;
raw_to->data_ = buffer;
}
template <typename T>
void CopyTypedDataBaseWithSafepointChecks(Thread* thread,
const T& from,
const T& to,
intptr_t length) {
constexpr intptr_t kChunkSize = 100 * 1024;
const intptr_t chunks = length / kChunkSize;
const intptr_t remainder = length % kChunkSize;
// Notice we re-load the data pointer, since T may be TypedData in which case
// the interior pointer may change after checking into safepoints.
for (intptr_t i = 0; i < chunks; ++i) {
memmove(to.ptr().untag()->data_ + i * kChunkSize,
from.ptr().untag()->data_ + i * kChunkSize, kChunkSize);
thread->CheckForSafepoint();
}
if (remainder > 0) {
memmove(to.ptr().untag()->data_ + chunks * kChunkSize,
from.ptr().untag()->data_ + chunks * kChunkSize, remainder);
}
}
void InitializeExternalTypedDataWithSafepointChecks(
Thread* thread,
intptr_t cid,
const ExternalTypedData& from,
const ExternalTypedData& to) {
const intptr_t length_in_elements = from.Length();
const intptr_t length_in_bytes =
TypedData::ElementSizeInBytes(cid) * length_in_elements;
uint8_t* to_data = static_cast<uint8_t*>(malloc(length_in_bytes));
to.ptr().untag()->data_ = to_data;
to.ptr().untag()->length_ = Smi::New(length_in_elements);
CopyTypedDataBaseWithSafepointChecks(thread, from, to, length_in_bytes);
}
void InitializeTypedDataView(TypedDataViewPtr obj) {
obj.untag()->typed_data_ = TypedDataBase::null();
obj.untag()->offset_in_bytes_ = Smi::New(0);
obj.untag()->length_ = Smi::New(0);
}
void FreeExternalTypedData(void* isolate_callback_data, void* buffer) {
free(buffer);
}
void FreeTransferablePeer(void* isolate_callback_data, void* peer) {
delete static_cast<TransferableTypedDataPeer*>(peer);
}
class SlowFromTo {
public:
explicit SlowFromTo(const GrowableObjectArray& storage) : storage_(storage) {}
ObjectPtr At(intptr_t index) { return storage_.At(index); }
void Add(const Object& key, const Object& value) {
storage_.Add(key);
storage_.Add(value);
}
intptr_t Length() { return storage_.Length(); }
private:
const GrowableObjectArray& storage_;
};
class FastFromTo {
public:
explicit FastFromTo(GrowableArray<ObjectPtr>& storage) : storage_(storage) {}
ObjectPtr At(intptr_t index) { return storage_.At(index); }
void Add(ObjectPtr key, ObjectPtr value) {
intptr_t i = storage_.length();
storage_.Resize(i + 2);
storage_[i + 0] = key;
storage_[i + 1] = value;
}
intptr_t Length() { return storage_.length(); }
private:
GrowableArray<ObjectPtr>& storage_;
};
static ObjectPtr Ptr(ObjectPtr obj) {
return obj;
}
static ObjectPtr Ptr(const Object& obj) {
return obj.ptr();
}
#if defined(HASH_IN_OBJECT_HEADER)
class IdentityMap {
public:
explicit IdentityMap(Thread* thread) : thread_(thread) {
hash_table_used_ = 0;
hash_table_capacity_ = 32;
hash_table_ = reinterpret_cast<uint32_t*>(
malloc(hash_table_capacity_ * sizeof(uint32_t)));
memset(hash_table_, 0, hash_table_capacity_ * sizeof(uint32_t));
}
~IdentityMap() { free(hash_table_); }
template <typename S, typename T>
DART_FORCE_INLINE ObjectPtr ForwardedObject(const S& object, T from_to) {
intptr_t mask = hash_table_capacity_ - 1;
intptr_t probe = GetHeaderHash(Ptr(object)) & mask;
for (;;) {
intptr_t index = hash_table_[probe];
if (index == 0) {
return Marker();
}
if (from_to.At(index) == Ptr(object)) {
return from_to.At(index + 1);
}
probe = (probe + 1) & mask;
}
}
template <typename S, typename T>
DART_FORCE_INLINE void Insert(const S& from,
const S& to,
T from_to,
bool check_for_safepoint) {
ASSERT(ForwardedObject(from, from_to) == Marker());
const auto id = from_to.Length();
from_to.Add(from, to); // Must occur before rehashing.
intptr_t mask = hash_table_capacity_ - 1;
intptr_t probe = GetHeaderHash(Ptr(from)) & mask;
for (;;) {
intptr_t index = hash_table_[probe];
if (index == 0) {
hash_table_[probe] = id;
break;
}
probe = (probe + 1) & mask;
}
hash_table_used_++;
if (hash_table_used_ * 2 > hash_table_capacity_) {
Rehash(hash_table_capacity_ * 2, from_to, check_for_safepoint);
}
}
private:
DART_FORCE_INLINE
uint32_t GetHeaderHash(ObjectPtr object) {
uint32_t hash = Object::GetCachedHash(object);
if (hash == 0) {
switch (object->GetClassIdOfHeapObject()) {
case kMintCid:
hash = Mint::Value(static_cast<MintPtr>(object));
// Don't write back: doesn't agree with dart:core's identityHash.
break;
case kDoubleCid:
hash =
bit_cast<uint64_t>(Double::Value(static_cast<DoublePtr>(object)));
// Don't write back: doesn't agree with dart:core's identityHash.
break;
case kOneByteStringCid:
case kTwoByteStringCid:
hash = String::Hash(static_cast<StringPtr>(object));
hash = Object::SetCachedHashIfNotSet(object, hash);
break;
default:
do {
hash = thread_->random()->NextUInt32();
} while (hash == 0 || !Smi::IsValid(hash));
hash = Object::SetCachedHashIfNotSet(object, hash);
break;
}
}
return hash;
}
template <typename T>
void Rehash(intptr_t new_capacity, T from_to, bool check_for_safepoint) {
hash_table_capacity_ = new_capacity;
hash_table_used_ = 0;
free(hash_table_);
hash_table_ = reinterpret_cast<uint32_t*>(
malloc(hash_table_capacity_ * sizeof(uint32_t)));
for (intptr_t i = 0; i < hash_table_capacity_; i++) {
hash_table_[i] = 0;
if (check_for_safepoint && (((i + 1) % kSlotsPerInterruptCheck) == 0)) {
thread_->CheckForSafepoint();
}
}
for (intptr_t id = 2; id < from_to.Length(); id += 2) {
ObjectPtr obj = from_to.At(id);
intptr_t mask = hash_table_capacity_ - 1;
intptr_t probe = GetHeaderHash(obj) & mask;
for (;;) {
if (hash_table_[probe] == 0) {
hash_table_[probe] = id;
hash_table_used_++;
break;
}
probe = (probe + 1) & mask;
}
if (check_for_safepoint && (((id + 2) % kSlotsPerInterruptCheck) == 0)) {
thread_->CheckForSafepoint();
}
}
}
Thread* thread_;
uint32_t* hash_table_;
uint32_t hash_table_capacity_;
uint32_t hash_table_used_;
};
#else // defined(HASH_IN_OBJECT_HEADER)
class IdentityMap {
public:
explicit IdentityMap(Thread* thread) : thread_(thread) {
thread_->set_forward_table_new(new WeakTable());
thread_->set_forward_table_old(new WeakTable());
}
~IdentityMap() {
thread_->set_forward_table_new(nullptr);
thread_->set_forward_table_old(nullptr);
}
template <typename S, typename T>
DART_FORCE_INLINE ObjectPtr ForwardedObject(const S& object, T from_to) {
const intptr_t id = GetObjectId(Ptr(object));
if (id == 0) return Marker();
return from_to.At(id + 1);
}
template <typename S, typename T>
DART_FORCE_INLINE void Insert(const S& from,
const S& to,
T from_to,
bool check_for_safepoint) {
ASSERT(ForwardedObject(from, from_to) == Marker());
const auto id = from_to.Length();
// May take >100ms and cannot yield to safepoints.
SetObjectId(Ptr(from), id);
from_to.Add(from, to);
}
private:
DART_FORCE_INLINE
intptr_t GetObjectId(ObjectPtr object) {
if (object->IsNewObject()) {
return thread_->forward_table_new()->GetValueExclusive(object);
} else {
return thread_->forward_table_old()->GetValueExclusive(object);
}
}
DART_FORCE_INLINE
void SetObjectId(ObjectPtr object, intptr_t id) {
if (object->IsNewObject()) {
thread_->forward_table_new()->SetValueExclusive(object, id);
} else {
thread_->forward_table_old()->SetValueExclusive(object, id);
}
}
Thread* thread_;
};
#endif // defined(HASH_IN_OBJECT_HEADER)
class ForwardMapBase {
public:
explicit ForwardMapBase(Thread* thread)
: thread_(thread), zone_(thread->zone()) {}
protected:
friend class ObjectGraphCopier;
void FinalizeTransferable(const TransferableTypedData& from,
const TransferableTypedData& to) {
// Get the old peer.
auto fpeer = static_cast<TransferableTypedDataPeer*>(
thread_->heap()->GetPeer(from.ptr()));
ASSERT(fpeer != nullptr && fpeer->data() != nullptr);
const intptr_t length = fpeer->length();
// Allocate new peer object with (data, length).
auto tpeer = new TransferableTypedDataPeer(fpeer->data(), length);
thread_->heap()->SetPeer(to.ptr(), tpeer);
// Move the handle itself to the new object.
fpeer->handle()->EnsureFreedExternal(thread_->isolate_group());
FinalizablePersistentHandle* finalizable_ref =
FinalizablePersistentHandle::New(thread_->isolate_group(), to, tpeer,
FreeTransferablePeer, length,
/*auto_delete=*/true);
ASSERT(finalizable_ref != nullptr);
tpeer->set_handle(finalizable_ref);
fpeer->ClearData();
}
void FinalizeExternalTypedData(const ExternalTypedData& to) {
to.AddFinalizer(to.DataAddr(0), &FreeExternalTypedData, to.LengthInBytes());
}
Thread* thread_;
Zone* zone_;
private:
DISALLOW_COPY_AND_ASSIGN(ForwardMapBase);
};
class FastForwardMap : public ForwardMapBase {
public:
explicit FastForwardMap(Thread* thread, IdentityMap* map)
: ForwardMapBase(thread),
map_(map),
raw_from_to_(thread->zone(), 20),
raw_transferables_from_to_(thread->zone(), 0),
raw_objects_to_rehash_(thread->zone(), 0),
raw_expandos_to_rehash_(thread->zone(), 0) {
raw_from_to_.Resize(2);
raw_from_to_[0] = Object::null();
raw_from_to_[1] = Object::null();
fill_cursor_ = 2;
}
ObjectPtr ForwardedObject(ObjectPtr object) {
return map_->ForwardedObject(object, FastFromTo(raw_from_to_));
}
void Insert(ObjectPtr from, ObjectPtr to, intptr_t size) {
map_->Insert(from, to, FastFromTo(raw_from_to_),
/*check_for_safepoint*/ false);
allocated_bytes += size;
}
void AddTransferable(TransferableTypedDataPtr from,
TransferableTypedDataPtr to) {
raw_transferables_from_to_.Add(from);
raw_transferables_from_to_.Add(to);
}
void AddWeakProperty(WeakPropertyPtr from) { raw_weak_properties_.Add(from); }
void AddWeakReference(WeakReferencePtr from) {
raw_weak_references_.Add(from);
}
void AddExternalTypedData(ExternalTypedDataPtr to) {
raw_external_typed_data_to_.Add(to);
}
void AddObjectToRehash(ObjectPtr to) { raw_objects_to_rehash_.Add(to); }
void AddExpandoToRehash(ObjectPtr to) { raw_expandos_to_rehash_.Add(to); }
private:
friend class FastObjectCopy;
friend class ObjectGraphCopier;
IdentityMap* map_;
GrowableArray<ObjectPtr> raw_from_to_;
GrowableArray<TransferableTypedDataPtr> raw_transferables_from_to_;
GrowableArray<ExternalTypedDataPtr> raw_external_typed_data_to_;
GrowableArray<ObjectPtr> raw_objects_to_rehash_;
GrowableArray<ObjectPtr> raw_expandos_to_rehash_;
GrowableArray<WeakPropertyPtr> raw_weak_properties_;
GrowableArray<WeakReferencePtr> raw_weak_references_;
intptr_t fill_cursor_ = 0;
intptr_t allocated_bytes = 0;
DISALLOW_COPY_AND_ASSIGN(FastForwardMap);
};
class SlowForwardMap : public ForwardMapBase {
public:
explicit SlowForwardMap(Thread* thread, IdentityMap* map)
: ForwardMapBase(thread),
map_(map),
from_to_transition_(thread->zone(), 2),
from_to_(GrowableObjectArray::Handle(thread->zone(),
GrowableObjectArray::New(2))),
transferables_from_to_(thread->zone(), 0) {
from_to_transition_.Resize(2);
from_to_transition_[0] = &PassiveObject::Handle();
from_to_transition_[1] = &PassiveObject::Handle();
from_to_.Add(Object::null_object());
from_to_.Add(Object::null_object());
fill_cursor_ = 2;
}
ObjectPtr ForwardedObject(ObjectPtr object) {
return map_->ForwardedObject(object, SlowFromTo(from_to_));
}
void Insert(const Object& from, const Object& to, intptr_t size) {
map_->Insert(from, to, SlowFromTo(from_to_),
/* check_for_safepoint */ true);
allocated_bytes += size;
}
void AddTransferable(const TransferableTypedData& from,
const TransferableTypedData& to) {
transferables_from_to_.Add(&TransferableTypedData::Handle(from.ptr()));
transferables_from_to_.Add(&TransferableTypedData::Handle(to.ptr()));
}
void AddWeakProperty(const WeakProperty& from) {
weak_properties_.Add(&WeakProperty::Handle(from.ptr()));
}
void AddWeakReference(const WeakReference& from) {
weak_references_.Add(&WeakReference::Handle(from.ptr()));
}
const ExternalTypedData& AddExternalTypedData(ExternalTypedDataPtr to) {
auto to_handle = &ExternalTypedData::Handle(to);
external_typed_data_.Add(to_handle);
return *to_handle;
}
void AddObjectToRehash(const Object& to) {
objects_to_rehash_.Add(&Object::Handle(to.ptr()));
}
void AddExpandoToRehash(const Object& to) {
expandos_to_rehash_.Add(&Object::Handle(to.ptr()));
}
void FinalizeTransferables() {
for (intptr_t i = 0; i < transferables_from_to_.length(); i += 2) {
auto from = transferables_from_to_[i];
auto to = transferables_from_to_[i + 1];
FinalizeTransferable(*from, *to);
}
}
void FinalizeExternalTypedData() {
for (intptr_t i = 0; i < external_typed_data_.length(); i++) {
auto to = external_typed_data_[i];
ForwardMapBase::FinalizeExternalTypedData(*to);
}
}
private:
friend class SlowObjectCopy;
friend class SlowObjectCopyBase;
friend class ObjectGraphCopier;
IdentityMap* map_;
GrowableArray<const PassiveObject*> from_to_transition_;
GrowableObjectArray& from_to_;
GrowableArray<const TransferableTypedData*> transferables_from_to_;
GrowableArray<const ExternalTypedData*> external_typed_data_;
GrowableArray<const Object*> objects_to_rehash_;
GrowableArray<const Object*> expandos_to_rehash_;
GrowableArray<const WeakProperty*> weak_properties_;
GrowableArray<const WeakReference*> weak_references_;
intptr_t fill_cursor_ = 0;
intptr_t allocated_bytes = 0;
DISALLOW_COPY_AND_ASSIGN(SlowForwardMap);
};
class ObjectCopyBase {
public:
explicit ObjectCopyBase(Thread* thread)
: thread_(thread),
heap_base_(thread->heap_base()),
zone_(thread->zone()),
heap_(thread->isolate_group()->heap()),
class_table_(thread->isolate_group()->class_table()),
new_space_(heap_->new_space()),
tmp_(Object::Handle(thread->zone())),
to_(Object::Handle(thread->zone())),
expando_cid_(Class::GetClassId(
thread->isolate_group()->object_store()->expando_class())),
exception_unexpected_object_(Object::Handle(thread->zone())) {}
~ObjectCopyBase() {}
protected:
static ObjectPtr LoadPointer(ObjectPtr src, intptr_t offset) {
return src.untag()->LoadPointer(reinterpret_cast<ObjectPtr*>(
reinterpret_cast<uint8_t*>(src.untag()) + offset));
}
static CompressedObjectPtr LoadCompressedPointer(ObjectPtr src,
intptr_t offset) {
return src.untag()->LoadPointer(reinterpret_cast<CompressedObjectPtr*>(
reinterpret_cast<uint8_t*>(src.untag()) + offset));
}
static compressed_uword LoadCompressedNonPointerWord(ObjectPtr src,
intptr_t offset) {
return *reinterpret_cast<compressed_uword*>(
reinterpret_cast<uint8_t*>(src.untag()) + offset);
}
static void StorePointerBarrier(ObjectPtr obj,
intptr_t offset,
ObjectPtr value) {
obj.untag()->StorePointer(
reinterpret_cast<ObjectPtr*>(reinterpret_cast<uint8_t*>(obj.untag()) +
offset),
value);
}
static void StoreCompressedPointerBarrier(ObjectPtr obj,
intptr_t offset,
ObjectPtr value) {
obj.untag()->StoreCompressedPointer(
reinterpret_cast<CompressedObjectPtr*>(
reinterpret_cast<uint8_t*>(obj.untag()) + offset),
value);
}
void StoreCompressedLargeArrayPointerBarrier(ObjectPtr obj,
intptr_t offset,
ObjectPtr value) {
obj.untag()->StoreCompressedArrayPointer(
reinterpret_cast<CompressedObjectPtr*>(
reinterpret_cast<uint8_t*>(obj.untag()) + offset),
value, thread_);
}
static void StorePointerNoBarrier(ObjectPtr obj,
intptr_t offset,
ObjectPtr value) {
*reinterpret_cast<ObjectPtr*>(reinterpret_cast<uint8_t*>(obj.untag()) +
offset) = value;
}
template <typename T = ObjectPtr>
static void StoreCompressedPointerNoBarrier(ObjectPtr obj,
intptr_t offset,
T value) {
*reinterpret_cast<CompressedObjectPtr*>(
reinterpret_cast<uint8_t*>(obj.untag()) + offset) = value;
}
static void StoreCompressedNonPointerWord(ObjectPtr obj,
intptr_t offset,
compressed_uword value) {
*reinterpret_cast<compressed_uword*>(
reinterpret_cast<uint8_t*>(obj.untag()) + offset) = value;
}
DART_FORCE_INLINE
bool CanCopyObject(uword tags, ObjectPtr object) {
const auto cid = UntaggedObject::ClassIdTag::decode(tags);
if (Class::IsIsolateUnsendable(class_table_->At(cid))) {
exception_msg_ = OS::SCreate(
zone_,
"Illegal argument in isolate message: object is unsendable - %s ("
"see restrictions listed at `SendPort.send()` documentation "
"for more information)",
Class::Handle(class_table_->At(cid)).ToCString());
exception_unexpected_object_ = object;
return false;
}
if (cid > kNumPredefinedCids) {
return true;
}
#define HANDLE_ILLEGAL_CASE(Type) \
case k##Type##Cid: { \
exception_msg_ = \
"Illegal argument in isolate message: " \
"(object is a " #Type ")"; \
exception_unexpected_object_ = object; \
return false; \
}
switch (cid) {
// From "dart:ffi" we handle only Pointer/DynamicLibrary specially, since
// those are the only non-abstract classes (so we avoid checking more cids
// here that cannot happen in reality)
HANDLE_ILLEGAL_CASE(DynamicLibrary)
HANDLE_ILLEGAL_CASE(Finalizer)
HANDLE_ILLEGAL_CASE(NativeFinalizer)
HANDLE_ILLEGAL_CASE(MirrorReference)
HANDLE_ILLEGAL_CASE(Pointer)
HANDLE_ILLEGAL_CASE(ReceivePort)
HANDLE_ILLEGAL_CASE(SuspendState)
HANDLE_ILLEGAL_CASE(UserTag)
default:
return true;
}
}
Thread* thread_;
uword heap_base_;
Zone* zone_;
Heap* heap_;
ClassTable* class_table_;
Scavenger* new_space_;
Object& tmp_;
Object& to_;
intptr_t expando_cid_;
const char* exception_msg_ = nullptr;
Object& exception_unexpected_object_;
};
class RetainingPath {
class Visitor : public ObjectPointerVisitor {
public:
Visitor(IsolateGroup* isolate_group,
RetainingPath* retaining_path,
MallocGrowableArray<ObjectPtr>* const working_list,
TraversalRules traversal_rules)
: ObjectPointerVisitor(isolate_group),
retaining_path_(retaining_path),
working_list_(working_list),
traversal_rules_(traversal_rules) {}
void VisitObject(ObjectPtr obj) {
if (!obj->IsHeapObject()) {
return;
}
// Skip canonical objects when rules are for messages internal to
// an isolate group. Otherwise, need to inspect canonical objects
// as well.
if (traversal_rules_ == TraversalRules::kInternalToIsolateGroup &&
obj->untag()->IsCanonical()) {
return;
}
if (retaining_path_->WasVisited(obj)) {
return;
}
retaining_path_->MarkVisited(obj);
working_list_->Add(obj);
}
void VisitPointers(ObjectPtr* from, ObjectPtr* to) override {
for (ObjectPtr* ptr = from; ptr <= to; ptr++) {
VisitObject(*ptr);
}
}
#if defined(DART_COMPRESSED_POINTERS)
void VisitCompressedPointers(uword heap_base,
CompressedObjectPtr* from,
CompressedObjectPtr* to) override {
for (CompressedObjectPtr* ptr = from; ptr <= to; ptr++) {
VisitObject(ptr->Decompress(heap_base));
}
}
#endif
RetainingPath* retaining_path_;
MallocGrowableArray<ObjectPtr>* const working_list_;
TraversalRules traversal_rules_;
};
public:
RetainingPath(Zone* zone,
Thread* thread,
const Object& from,
const Object& to,
TraversalRules traversal_rules)
: zone_(zone),
thread_(thread),
from_(from),
to_(to),
traversal_rules_(traversal_rules) {
thread_->set_forward_table_new(new WeakTable());
thread_->set_forward_table_old(new WeakTable());
}
~RetainingPath() {
thread_->set_forward_table_new(nullptr);
thread_->set_forward_table_old(nullptr);
}
bool WasVisited(ObjectPtr object) {
if (object->IsNewObject()) {
return thread_->forward_table_new()->GetValueExclusive(object) != 0;
} else {
return thread_->forward_table_old()->GetValueExclusive(object) != 0;
}
}
void MarkVisited(ObjectPtr object) {
if (object->IsNewObject()) {
thread_->forward_table_new()->SetValueExclusive(object, 1);
} else {
thread_->forward_table_old()->SetValueExclusive(object, 1);
}
}
const char* FindPath() {
MallocGrowableArray<ObjectPtr>* const working_list =
thread_->pointers_to_verify_at_exit();
ASSERT(working_list->length() == 0);
Visitor visitor(thread_->isolate_group(), this, working_list,
traversal_rules_);
MarkVisited(from_.ptr());
working_list->Add(from_.ptr());
Thread* thread = Thread::Current();
ClassTable* class_table = thread->isolate_group()->class_table();
Closure& closure = Closure::Handle(zone_);
Array& array = Array::Handle(zone_);
Class& klass = Class::Handle(zone_);
while (!working_list->is_empty()) {
thread->CheckForSafepoint();
// Keep node in the list, separated by null value so that
// if we are to add children, children can find it in case
// they are on retaining path.
ObjectPtr raw = working_list->Last();
if (raw == Object::null()) {
// If all children of a node were processed, then skip the separator,
working_list->RemoveLast();
// then skip the parent since it has already been processed too.
working_list->RemoveLast();
continue;
}
if (raw == to_.ptr()) {
return CollectPath(working_list);
}
// Separator null object indicates children goes next in the working_list
working_list->Add(Object::null());
int length = working_list->length();
do { // This loop is here so that we can skip children processing
const intptr_t cid = raw->GetClassIdOfHeapObject();
if (traversal_rules_ == TraversalRules::kInternalToIsolateGroup) {
if (CanShareObjectAcrossIsolates(raw)) {
break;
}
if (cid == kClosureCid) {
closure ^= raw;
// Only context has to be checked.
working_list->Add(closure.RawContext());
break;
}
// These we are not expected to drill into as they can't be on
// retaining path, they are illegal to send.
klass = class_table->At(cid);
if (klass.is_isolate_unsendable()) {
break;
}
} else {
ASSERT(traversal_rules_ ==
TraversalRules::kExternalBetweenIsolateGroups);
// Skip classes that are illegal to send across isolate groups.
// (keep the list in sync with message_snapshot.cc)
bool skip = false;
switch (cid) {
case kClosureCid:
case kFinalizerCid:
case kFinalizerEntryCid:
case kFunctionTypeCid:
case kMirrorReferenceCid:
case kNativeFinalizerCid:
case kReceivePortCid:
case kRecordCid:
case kRecordTypeCid:
case kRegExpCid:
case kStackTraceCid:
case kSuspendStateCid:
case kUserTagCid:
case kWeakPropertyCid:
case kWeakReferenceCid:
case kWeakArrayCid:
case kDynamicLibraryCid:
case kPointerCid:
case kInstanceCid:
skip = true;
break;
default:
if (cid >= kNumPredefinedCids) {
skip = true;
}
}
if (skip) {
break;
}
}
if (cid == kArrayCid) {
array ^= Array::RawCast(raw);
visitor.VisitObject(array.GetTypeArguments());
const intptr_t batch_size = (2 << 14) - 1;
for (intptr_t i = 0; i < array.Length(); ++i) {
ObjectPtr ptr = array.At(i);
visitor.VisitObject(ptr);
if ((i & batch_size) == batch_size) {
thread->CheckForSafepoint();
}
}
break;
} else {
raw->untag()->VisitPointers(&visitor);
}
} while (false);
// If no children were added, remove null separator and the node.
// If children were added, the node will be removed once last child
// is processed, only separator null remains.
if (working_list->length() == length) {
RELEASE_ASSERT(working_list->RemoveLast() == Object::null());
RELEASE_ASSERT(working_list->RemoveLast() == raw);
}
}
// `to` was not found in the graph rooted in `from`, empty retaining path
return "";
}
private:
Zone* zone_;
Thread* thread_;
const Object& from_;
const Object& to_;
TraversalRules traversal_rules_;
class FindObjectVisitor : public ObjectPointerVisitor {
public:
FindObjectVisitor(IsolateGroup* isolate_group, ObjectPtr target)
: ObjectPointerVisitor(isolate_group), target_(target), index_(0) {}
void VisitPointers(ObjectPtr* from, ObjectPtr* to) override {
for (ObjectPtr* ptr = from; ptr <= to; ptr++, index_++) {
if (*ptr == target_) {
break;
}
}
}
#if defined(DART_COMPRESSED_POINTERS)
void VisitCompressedPointers(uword heap_base,
CompressedObjectPtr* from,
CompressedObjectPtr* to) override {
for (CompressedObjectPtr* ptr = from; ptr <= to; ptr++, index_++) {
if (ptr->Decompress(heap_base) == target_) {
break;
}
}
}
#endif
intptr_t index() { return index_; }
private:
ObjectPtr target_;
intptr_t index_;
};
const char* CollectPath(MallocGrowableArray<ObjectPtr>* const working_list) {
Object& previous_object = Object::Handle(zone_);
Object& object = Object::Handle(zone_);
Field& field = Field::Handle(zone_);
Class& klass = Class::Handle(zone_);
Library& library = Library::Handle(zone_);
String& library_url = String::Handle(zone_);
Context& context = Context::Handle(zone_);
Closure& closure = Closure::Handle(zone_);
Function& function = Function::Handle(zone_);
#if !defined(DART_PRECOMPILED_RUNTIME)
Code& code = Code::Handle(zone_);
LocalVarDescriptors& var_descriptors = LocalVarDescriptors::Handle(zone_);
String& name = String::Handle(zone_);
#endif
const char* saved_context_location = nullptr;
intptr_t saved_context_object_index = -1;
intptr_t saved_context_depth = 0;
const char* retaining_path = "";
ObjectPtr raw = to_.ptr();
do {
previous_object = raw;
// Skip all remaining children until null-separator, so we get the parent
do {
raw = working_list->RemoveLast();
} while (raw != Object::null() && raw != from_.ptr());
if (raw == Object::null()) {
raw = working_list->RemoveLast();
object = raw;
klass = object.clazz();
const char* location = object.ToCString();
if (object.IsContext()) {
context ^= raw;
if (saved_context_object_index == -1) {
// If this is the first context, remember index of the
// [previous_object] in the Context.
// We will need it later if get to see the Closure next.
saved_context_depth = 0;
for (intptr_t i = 0; i < context.num_variables(); i++) {
if (context.At(i) == previous_object.ptr()) {
saved_context_object_index = i;
break;
}
}
} else {
// Keep track of context depths in case of nested contexts;
saved_context_depth++;
}
} else {
if (object.IsInstance()) {
if (object.IsClosure()) {
closure ^= raw;
function ^= closure.function();
// Use function's class when looking for a library information.
klass ^= function.Owner();
#if defined(DART_PRECOMPILED_RUNTIME)
// Use function's name instead of closure's.
location = function.QualifiedUserVisibleNameCString();
#else // defined(DART_PRECOMPILED_RUNTIME) \
// Attempt to convert "instance <- Context+ <- Closure" into \
// "instance <- local var name in Closure".
if (!function.ForceOptimize()) {
function.EnsureHasCompiledUnoptimizedCode();
}
code ^= function.unoptimized_code();
ASSERT(!code.IsNull());
var_descriptors ^= code.GetLocalVarDescriptors();
for (intptr_t i = 0; i < var_descriptors.Length(); i++) {
UntaggedLocalVarDescriptors::VarInfo info;
var_descriptors.GetInfo(i, &info);
if (info.scope_id == -saved_context_depth &&
info.kind() ==
UntaggedLocalVarDescriptors::VarInfoKind::kContextVar &&
info.index() == saved_context_object_index) {
name ^= var_descriptors.GetName(i);
location =
OS::SCreate(zone_, "field %s in %s", name.ToCString(),
function.QualifiedUserVisibleNameCString());
// Won't need saved context location after all.
saved_context_location = nullptr;
break;
}
}
#endif // defined(DART_PRECOMPILED_RUNTIME)
} else {
// Attempt to find field name for the field that holds the
// [previous_object] instance.
FindObjectVisitor visitor(thread_->isolate_group(),
previous_object.ptr());
raw->untag()->VisitPointers(&visitor);
field ^= klass.FieldFromIndex(visitor.index());
if (!field.IsNull()) {
location =
OS::SCreate(zone_, "%s in %s",
field.UserVisibleNameCString(), location);
}
}
}
// Saved context object index stays up for only one cycle - just to
// accommodate short chains Closure -> Context -> instance.
saved_context_object_index = -1;
saved_context_depth = -1;
}
// Add library url to the location if library is available.
library = klass.library();
if (!library.IsNull()) {
library_url = library.url();
location = OS::SCreate(zone_, "%s (from %s)", location,
library_url.ToCString());
}
if (object.IsContext()) {
// Save context string placeholder in case we don't find closure next
if (saved_context_location == nullptr) {
saved_context_location = location;
} else {
// Append saved contexts
saved_context_location = OS::SCreate(
zone_, "%s <- %s\n", saved_context_location, location);
}
} else {
if (saved_context_location != nullptr) {
// Could not use saved context, insert it into retaining path now.
retaining_path = OS::SCreate(zone_, "%s <- %s", retaining_path,
saved_context_location);
saved_context_location = nullptr;
}
retaining_path =
OS::SCreate(zone_, "%s <- %s\n", retaining_path, location);
}
}
} while (raw != from_.ptr());
ASSERT(working_list->is_empty());
return retaining_path;
}
};
const char* FindRetainingPath(Zone* zone_,
Thread* thread,
const Object& from,
const Object& to,
TraversalRules traversal_rules) {
RetainingPath rr(zone_, thread, from, to, traversal_rules);
return rr.FindPath();
}
class FastObjectCopyBase : public ObjectCopyBase {
public:
using Types = PtrTypes;
FastObjectCopyBase(Thread* thread, IdentityMap* map)
: ObjectCopyBase(thread), fast_forward_map_(thread, map) {}
protected:
DART_FORCE_INLINE
void ForwardCompressedPointers(ObjectPtr src,
ObjectPtr dst,
intptr_t offset,
intptr_t end_offset) {
for (; offset < end_offset; offset += kCompressedWordSize) {
ForwardCompressedPointer(src, dst, offset);
}
}
DART_FORCE_INLINE
void ForwardCompressedPointers(ObjectPtr src,
ObjectPtr dst,
intptr_t offset,
intptr_t end_offset,
UnboxedFieldBitmap bitmap) {
if (bitmap.IsEmpty()) {
ForwardCompressedPointers(src, dst, offset, end_offset);
return;
}
intptr_t bit = offset >> kCompressedWordSizeLog2;
for (; offset < end_offset; offset += kCompressedWordSize) {
if (bitmap.Get(bit++)) {
StoreCompressedNonPointerWord(
dst, offset, LoadCompressedNonPointerWord(src, offset));
} else {
ForwardCompressedPointer(src, dst, offset);
}
}
}
void ForwardCompressedArrayPointers(intptr_t array_length,
ObjectPtr src,
ObjectPtr dst,
intptr_t offset,
intptr_t end_offset) {
for (; offset < end_offset; offset += kCompressedWordSize) {
ForwardCompressedPointer(src, dst, offset);
}
}
void ForwardCompressedContextPointers(intptr_t context_length,
ObjectPtr src,
ObjectPtr dst,
intptr_t offset,
intptr_t end_offset) {
for (; offset < end_offset; offset += kCompressedWordSize) {
ForwardCompressedPointer(src, dst, offset);
}
}
DART_FORCE_INLINE
void ForwardCompressedPointer(ObjectPtr src, ObjectPtr dst, intptr_t offset) {
auto value = LoadCompressedPointer(src, offset);
if (!value.IsHeapObject()) {
StoreCompressedPointerNoBarrier(dst, offset, value);
return;
}
auto value_decompressed = value.Decompress(heap_base_);
const uword tags = TagsFromUntaggedObject(value_decompressed.untag());
if (CanShareObject(value_decompressed, tags)) {
StoreCompressedPointerNoBarrier(dst, offset, value);
return;
}
ObjectPtr existing_to =
fast_forward_map_.ForwardedObject(value_decompressed);
if (existing_to != Marker()) {
StoreCompressedPointerNoBarrier(dst, offset, existing_to);
return;
}
if (UNLIKELY(!CanCopyObject(tags, value_decompressed))) {
ASSERT(exception_msg_ != nullptr);
StoreCompressedPointerNoBarrier(dst, offset, Object::null());
return;
}
auto to = Forward(tags, value_decompressed);
StoreCompressedPointerNoBarrier(dst, offset, to);
}
ObjectPtr Forward(uword tags, ObjectPtr from) {
const intptr_t header_size = UntaggedObject::SizeTag::decode(tags);
const auto cid = UntaggedObject::ClassIdTag::decode(tags);
const uword size =
header_size != 0 ? header_size : from.untag()->HeapSize();
if (IsAllocatableInNewSpace(size)) {
const uword alloc = new_space_->TryAllocateNoSafepoint(thread_, size);
if (alloc != 0) {
ObjectPtr to(reinterpret_cast<UntaggedObject*>(alloc));
fast_forward_map_.Insert(from, to, size);
if (IsExternalTypedDataClassId(cid)) {
SetNewSpaceTaggingWord(to, cid, header_size);
InitializeExternalTypedData(cid, ExternalTypedData::RawCast(from),
ExternalTypedData::RawCast(to));
fast_forward_map_.AddExternalTypedData(
ExternalTypedData::RawCast(to));
} else if (IsTypedDataViewClassId(cid) ||
IsUnmodifiableTypedDataViewClassId(cid)) {
// We set the views backing store to `null` to satisfy an assertion in
// GCCompactor::VisitTypedDataViewPointers().
SetNewSpaceTaggingWord(to, cid, header_size);
InitializeTypedDataView(TypedDataView::RawCast(to));
}
return to;
}
}
exception_msg_ = kFastAllocationFailed;
return Marker();
}
void EnqueueTransferable(TransferableTypedDataPtr from,
TransferableTypedDataPtr to) {
fast_forward_map_.AddTransferable(from, to);
}
void EnqueueWeakProperty(WeakPropertyPtr from) {
fast_forward_map_.AddWeakProperty(from);
}
void EnqueueWeakReference(WeakReferencePtr from) {
fast_forward_map_.AddWeakReference(from);
}
void EnqueueObjectToRehash(ObjectPtr to) {
fast_forward_map_.AddObjectToRehash(to);
}
void EnqueueExpandoToRehash(ObjectPtr to) {
fast_forward_map_.AddExpandoToRehash(to);
}
static void StoreCompressedArrayPointers(intptr_t array_length,
ObjectPtr src,
ObjectPtr dst,
intptr_t offset,
intptr_t end_offset) {
StoreCompressedPointers(src, dst, offset, end_offset);
}
static void StoreCompressedPointers(ObjectPtr src,
ObjectPtr dst,
intptr_t offset,
intptr_t end_offset) {
StoreCompressedPointersNoBarrier(src, dst, offset, end_offset);
}
static void StoreCompressedPointersNoBarrier(ObjectPtr src,
ObjectPtr dst,
intptr_t offset,
intptr_t end_offset) {
for (; offset <= end_offset; offset += kCompressedWordSize) {
StoreCompressedPointerNoBarrier(dst, offset,
LoadCompressedPointer(src, offset));
}
}
protected:
friend class ObjectGraphCopier;
FastForwardMap fast_forward_map_;
};
class SlowObjectCopyBase : public ObjectCopyBase {
public:
using Types = HandleTypes;
explicit SlowObjectCopyBase(Thread* thread, IdentityMap* map)
: ObjectCopyBase(thread), slow_forward_map_(thread, map) {}
protected:
DART_FORCE_INLINE
void ForwardCompressedPointers(const Object& src,
const Object& dst,
intptr_t offset,
intptr_t end_offset) {
for (; offset < end_offset; offset += kCompressedWordSize) {
ForwardCompressedPointer(src, dst, offset);
}
}
DART_FORCE_INLINE
void ForwardCompressedPointers(const Object& src,
const Object& dst,
intptr_t offset,
intptr_t end_offset,
UnboxedFieldBitmap bitmap) {
intptr_t bit = offset >> kCompressedWordSizeLog2;
for (; offset < end_offset; offset += kCompressedWordSize) {
if (bitmap.Get(bit++)) {
StoreCompressedNonPointerWord(
dst.ptr(), offset, LoadCompressedNonPointerWord(src.ptr(), offset));
} else {
ForwardCompressedPointer(src, dst, offset);
}
}
}
void ForwardCompressedArrayPointers(intptr_t array_length,
const Object& src,
const Object& dst,
intptr_t offset,
intptr_t end_offset) {
if (Array::UseCardMarkingForAllocation(array_length)) {
for (; offset < end_offset; offset += kCompressedWordSize) {
ForwardCompressedLargeArrayPointer(src, dst, offset);
thread_->CheckForSafepoint();
}
} else {
for (; offset < end_offset; offset += kCompressedWordSize) {
ForwardCompressedPointer(src, dst, offset);
}
}
}
void ForwardCompressedContextPointers(intptr_t context_length,
const Object& src,
const Object& dst,
intptr_t offset,
intptr_t end_offset) {
for (; offset < end_offset; offset += kCompressedWordSize) {
ForwardCompressedPointer(src, dst, offset);
}
}
DART_FORCE_INLINE
void ForwardCompressedLargeArrayPointer(const Object& src,
const Object& dst,
intptr_t offset) {
auto value = LoadCompressedPointer(src.ptr(), offset);
if (!value.IsHeapObject()) {
StoreCompressedPointerNoBarrier(dst.ptr(), offset, value);
return;
}
auto value_decompressed = value.Decompress(heap_base_);
const uword tags = TagsFromUntaggedObject(value_decompressed.untag());
if (CanShareObject(value_decompressed, tags)) {
StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset,
value_decompressed);
return;
}
ObjectPtr existing_to =
slow_forward_map_.ForwardedObject(value_decompressed);
if (existing_to != Marker()) {
StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset, existing_to);
return;
}
if (UNLIKELY(!CanCopyObject(tags, value_decompressed))) {
ASSERT(exception_msg_ != nullptr);
StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset,
Object::null());
return;
}
tmp_ = value_decompressed;
tmp_ = Forward(tags, tmp_); // Only this can cause allocation.
StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset, tmp_.ptr());
}
DART_FORCE_INLINE
void ForwardCompressedPointer(const Object& src,
const Object& dst,
intptr_t offset) {
auto value = LoadCompressedPointer(src.ptr(), offset);
if (!value.IsHeapObject()) {
StoreCompressedPointerNoBarrier(dst.ptr(), offset, value);
return;
}
auto value_decompressed = value.Decompress(heap_base_);
const uword tags = TagsFromUntaggedObject(value_decompressed.untag());
if (CanShareObject(value_decompressed, tags)) {
StoreCompressedPointerBarrier(dst.ptr(), offset, value_decompressed);
return;
}
ObjectPtr existing_to =
slow_forward_map_.ForwardedObject(value_decompressed);
if (existing_to != Marker()) {
StoreCompressedPointerBarrier(dst.ptr(), offset, existing_to);
return;
}
if (UNLIKELY(!CanCopyObject(tags, value_decompressed))) {
ASSERT(exception_msg_ != nullptr);
StoreCompressedPointerNoBarrier(dst.ptr(), offset, Object::null());
return;
}
tmp_ = value_decompressed;
tmp_ = Forward(tags, tmp_); // Only this can cause allocation.
StoreCompressedPointerBarrier(dst.ptr(), offset, tmp_.ptr());
}
ObjectPtr Forward(uword tags, const Object& from) {
const intptr_t cid = UntaggedObject::ClassIdTag::decode(tags);
intptr_t size = UntaggedObject::SizeTag::decode(tags);
if (size == 0) {
size = from.ptr().untag()->HeapSize();
}
to_ = AllocateObject(cid, size, slow_forward_map_.allocated_bytes);
UpdateLengthField(cid, from.ptr(), to_.ptr());
slow_forward_map_.Insert(from, to_, size);
ObjectPtr to = to_.ptr();
if ((cid == kArrayCid || cid == kImmutableArrayCid) &&
!IsAllocatableInNewSpace(size)) {
to.untag()->SetCardRememberedBitUnsynchronized();
Page::Of(to)->AllocateCardTable();
}
if (IsExternalTypedDataClassId(cid)) {
const auto& external_to = slow_forward_map_.AddExternalTypedData(
ExternalTypedData::RawCast(to));
InitializeExternalTypedDataWithSafepointChecks(
thread_, cid, ExternalTypedData::Cast(from), external_to);
return external_to.ptr();
} else if (IsTypedDataViewClassId(cid) ||
IsUnmodifiableTypedDataViewClassId(cid)) {
// We set the views backing store to `null` to satisfy an assertion in
// GCCompactor::VisitTypedDataViewPointers().
InitializeTypedDataView(TypedDataView::RawCast(to));
}
return to;
}
void EnqueueTransferable(const TransferableTypedData& from,
const TransferableTypedData& to) {
slow_forward_map_.AddTransferable(from, to);
}
void EnqueueWeakProperty(const WeakProperty& from) {
slow_forward_map_.AddWeakProperty(from);
}
void EnqueueWeakReference(const WeakReference& from) {
slow_forward_map_.AddWeakReference(from);
}
void EnqueueObjectToRehash(const Object& to) {
slow_forward_map_.AddObjectToRehash(to);
}
void EnqueueExpandoToRehash(const Object& to) {
slow_forward_map_.AddExpandoToRehash(to);
}
void StoreCompressedArrayPointers(intptr_t array_length,
const Object& src,
const Object& dst,
intptr_t offset,
intptr_t end_offset) {
auto src_ptr = src.ptr();
auto dst_ptr = dst.ptr();
if (Array::UseCardMarkingForAllocation(array_length)) {
for (; offset <= end_offset; offset += kCompressedWordSize) {
StoreCompressedLargeArrayPointerBarrier(
dst_ptr, offset,
LoadCompressedPointer(src_ptr, offset).Decompress(heap_base_));
}
} else {
for (; offset <= end_offset; offset += kCompressedWordSize) {
StoreCompressedPointerBarrier(
dst_ptr, offset,
LoadCompressedPointer(src_ptr, offset).Decompress(heap_base_));
}
}
}
void StoreCompressedPointers(const Object& src,
const Object& dst,
intptr_t offset,
intptr_t end_offset) {
auto src_ptr = src.ptr();
auto dst_ptr = dst.ptr();
for (; offset <= end_offset; offset += kCompressedWordSize) {
StoreCompressedPointerBarrier(
dst_ptr, offset,
LoadCompressedPointer(src_ptr, offset).Decompress(heap_base_));
}
}
static void StoreCompressedPointersNoBarrier(const Object& src,
const Object& dst,
intptr_t offset,
intptr_t end_offset) {
auto src_ptr = src.ptr();
auto dst_ptr = dst.ptr();
for (; offset <= end_offset; offset += kCompressedWordSize) {
StoreCompressedPointerNoBarrier(dst_ptr, offset,
LoadCompressedPointer(src_ptr, offset));
}
}
protected:
friend class ObjectGraphCopier;
SlowForwardMap slow_forward_map_;
};
template <typename Base>
class ObjectCopy : public Base {
public:
using Types = typename Base::Types;
ObjectCopy(Thread* thread, IdentityMap* map) : Base(thread, map) {}
void CopyPredefinedInstance(typename Types::Object from,
typename Types::Object to,
intptr_t cid) {
if (IsImplicitFieldClassId(cid)) {
CopyUserdefinedInstanceWithoutUnboxedFields(from, to);
return;
}
switch (cid) {
#define COPY_TO(clazz) \
case clazz::kClassId: { \
typename Types::clazz casted_from = Types::Cast##clazz(from); \
typename Types::clazz casted_to = Types::Cast##clazz(to); \
Copy##clazz(casted_from, casted_to); \
return; \
}
CLASS_LIST_NO_OBJECT_NOR_STRING_NOR_ARRAY_NOR_MAP(COPY_TO)
COPY_TO(Array)
COPY_TO(GrowableObjectArray)
COPY_TO(Map)
COPY_TO(Set)
#undef COPY_TO
case ImmutableArray::kClassId: {
typename Types::Array casted_from = Types::CastArray(from);
typename Types::Array casted_to = Types::CastArray(to);
CopyArray(casted_from, casted_to);
return;
}
#define COPY_TO(clazz) case kTypedData##clazz##Cid:
CLASS_LIST_TYPED_DATA(COPY_TO) {
typename Types::TypedData casted_from = Types::CastTypedData(from);
typename Types::TypedData casted_to = Types::CastTypedData(to);
CopyTypedData(casted_from, casted_to);
return;
}
#undef COPY_TO
case kByteDataViewCid:
case kUnmodifiableByteDataViewCid:
#define COPY_TO(clazz) \
case kTypedData##clazz##ViewCid: \
case kUnmodifiableTypedData##clazz##ViewCid:
CLASS_LIST_TYPED_DATA(COPY_TO) {
typename Types::TypedDataView casted_from =
Types::CastTypedDataView(from);
typename Types::TypedDataView casted_to =
Types::CastTypedDataView(to);
CopyTypedDataView(casted_from, casted_to);
return;
}
#undef COPY_TO
#define COPY_TO(clazz) case kExternalTypedData##clazz##Cid:
CLASS_LIST_TYPED_DATA(COPY_TO) {
typename Types::ExternalTypedData casted_from =
Types::CastExternalTypedData(from);
typename Types::ExternalTypedData casted_to =
Types::CastExternalTypedData(to);
CopyExternalTypedData(casted_from, casted_to);
return;
}
#undef COPY_TO
default:
break;
}
const Object& obj = Types::HandlifyObject(from);
FATAL("Unexpected object: %s\n", obj.ToCString());
}
void CopyUserdefinedInstance(typename Types::Object from,
typename Types::Object to,
UnboxedFieldBitmap bitmap) {
const intptr_t instance_size = UntagObject(from)->HeapSize();
Base::ForwardCompressedPointers(from, to, kWordSize, instance_size, bitmap);
}
void CopyUserdefinedInstanceWithoutUnboxedFields(typename Types::Object from,
typename Types::Object to) {
const intptr_t instance_size = UntagObject(from)->HeapSize();
Base::ForwardCompressedPointers(from, to, kWordSize, instance_size);
}
void CopyClosure(typename Types::Closure from, typename Types::Closure to) {
Base::StoreCompressedPointers(
from, to, OFFSET_OF(UntaggedClosure, instantiator_type_arguments_),
OFFSET_OF(UntaggedClosure, function_));
Base::ForwardCompressedPointer(from, to,
OFFSET_OF(UntaggedClosure, context_));
Base::StoreCompressedPointersNoBarrier(from, to,
OFFSET_OF(UntaggedClosure, hash_),
OFFSET_OF(UntaggedClosure, hash_));
ONLY_IN_PRECOMPILED(UntagClosure(to)->entry_point_ =
UntagClosure(from)->entry_point_);
}
void CopyContext(typename Types::Context from, typename Types::Context to) {
const intptr_t length = Context::NumVariables(Types::GetContextPtr(from));
UntagContext(to)->num_variables_ = UntagContext(from)->num_variables_;
Base::ForwardCompressedPointer(from, to,
OFFSET_OF(UntaggedContext, parent_));
Base::ForwardCompressedContextPointers(
length, from, to, Context::variable_offset(0),
Context::variable_offset(0) + Context::kBytesPerElement * length);
}
void CopyArray(typename Types::Array from, typename Types::Array to) {
const intptr_t length = Smi::Value(UntagArray(from)->length());
Base::StoreCompressedArrayPointers(
length, from, to, OFFSET_OF(UntaggedArray, type_arguments_),
OFFSET_OF(UntaggedArray, type_arguments_));
Base::StoreCompressedPointersNoBarrier(from, to,
OFFSET_OF(UntaggedArray, length_),
OFFSET_OF(UntaggedArray, length_));
Base::ForwardCompressedArrayPointers(
length, from, to, Array::data_offset(),
Array::data_offset() + kCompressedWordSize * length);
}
void CopyGrowableObjectArray(typename Types::GrowableObjectArray from,
typename Types::GrowableObjectArray to) {
Base::StoreCompressedPointers(
from, to, OFFSET_OF(UntaggedGrowableObjectArray, type_arguments_),
OFFSET_OF(UntaggedGrowableObjectArray, type_arguments_));
Base::StoreCompressedPointersNoBarrier(
from, to, OFFSET_OF(UntaggedGrowableObjectArray, length_),
OFFSET_OF(UntaggedGrowableObjectArray, length_));
Base::ForwardCompressedPointer(
from, to, OFFSET_OF(UntaggedGrowableObjectArray, data_));
}
void CopyRecord(typename Types::Record from, typename Types::Record to) {
const intptr_t num_fields = Record::NumFields(Types::GetRecordPtr(from));
Base::StoreCompressedPointersNoBarrier(from, to,
OFFSET_OF(UntaggedRecord, shape_),
OFFSET_OF(UntaggedRecord, shape_));
Base::ForwardCompressedPointers(
from, to, Record::field_offset(0),
Record::field_offset(0) + Record::kBytesPerElement * num_fields);
}
template <intptr_t one_for_set_two_for_map, typename T>
void CopyLinkedHashBase(T from,
T to,
UntaggedLinkedHashBase* from_untagged,
UntaggedLinkedHashBase* to_untagged) {
// We have to find out whether the map needs re-hashing on the receiver side
// due to keys being copied and the keys therefore possibly having different
// hash codes (e.g. due to user-defined hashCode implementation or due to
// new identity hash codes of the copied objects).
bool needs_rehashing = false;
ArrayPtr data = from_untagged->data_.Decompress(Base::heap_base_);
if (data != Array::null()) {
UntaggedArray* untagged_data = data.untag();
const intptr_t length = Smi::Value(untagged_data->length_);
auto key_value_pairs = untagged_data->data();
for (intptr_t i = 0; i < length; i += one_for_set_two_for_map) {
ObjectPtr key = key_value_pairs[i].Decompress(Base::heap_base_);
const bool is_deleted_entry = key == data;
if (key->IsHeapObject()) {
if (!is_deleted_entry && MightNeedReHashing(key)) {
needs_rehashing = true;
break;
}
}
}
}
Base::StoreCompressedPointers(
from, to, OFFSET_OF(UntaggedLinkedHashBase, type_arguments_),
OFFSET_OF(UntaggedLinkedHashBase, type_arguments_));
// Compared with the snapshot-based (de)serializer we do preserve the same
// backing store (i.e. used_data/deleted_keys/data) and therefore do not
// magically shrink backing store based on usage.
//
// We do this to avoid making assumptions about the object graph and the
// linked hash map (e.g. assuming there's no other references to the data,
// assuming the linked hashmap is in a consistent state)
if (needs_rehashing) {
to_untagged->hash_mask_ = Smi::New(0);
to_untagged->index_ = TypedData::RawCast(Object::null());
to_untagged->deleted_keys_ = Smi::New(0);
}
// From this point on we shouldn't use the raw pointers, since GC might
// happen when forwarding objects.
from_untagged = nullptr;
to_untagged = nullptr;
if (!needs_rehashing) {
Base::ForwardCompressedPointer(from, to,
OFFSET_OF(UntaggedLinkedHashBase, index_));
Base::StoreCompressedPointersNoBarrier(
from, to, OFFSET_OF(UntaggedLinkedHashBase, hash_mask_),
OFFSET_OF(UntaggedLinkedHashBase, hash_mask_));
Base::StoreCompressedPointersNoBarrier(
from, to, OFFSET_OF(UntaggedMap, deleted_keys_),
OFFSET_OF(UntaggedMap, deleted_keys_));
}
Base::ForwardCompressedPointer(from, to,
OFFSET_OF(UntaggedLinkedHashBase, data_));
Base::StoreCompressedPointersNoBarrier(
from, to, OFFSET_OF(UntaggedLinkedHashBase, used_data_),
OFFSET_OF(UntaggedLinkedHashBase, used_data_));
if (Base::exception_msg_ == nullptr && needs_rehashing) {
Base::EnqueueObjectToRehash(to);
}
}
void CopyMap(typename Types::Map from, typename Types::Map to) {
CopyLinkedHashBase<2, typename Types::Map>(from, to, UntagMap(from),
UntagMap(to));
}
void CopySet(typename Types::Set from, typename Types::Set to) {
CopyLinkedHashBase<1, typename Types::Set>(from, to, UntagSet(from),
UntagSet(to));
}
void CopyDouble(typename Types::Double from, typename Types::Double to) {
#if !defined(DART_PRECOMPILED_RUNTIME)
auto raw_from = UntagDouble(from);
auto raw_to = UntagDouble(to);
raw_to->value_ = raw_from->value_;
#else
// Will be shared and not copied.
UNREACHABLE();
#endif
}
void CopyFloat32x4(typename Types::Float32x4 from,
typename Types::Float32x4 to) {
#if !defined(DART_PRECOMPILED_RUNTIME)
auto raw_from = UntagFloat32x4(from);
auto raw_to = UntagFloat32x4(to);
raw_to->value_[0] = raw_from->value_[0];
raw_to->value_[1] = raw_from->value_[1];
raw_to->value_[2] = raw_from->value_[2];
raw_to->value_[3] = raw_from->value_[3];
#else
// Will be shared and not copied.
UNREACHABLE();
#endif
}
void CopyFloat64x2(typename Types::Float64x2 from,
typename Types::Float64x2 to) {
#if !defined(DART_PRECOMPILED_RUNTIME)
auto raw_from = UntagFloat64x2(from);
auto raw_to = UntagFloat64x2(to);
raw_to->value_[0] = raw_from->value_[0];
raw_to->value_[1] = raw_from->value_[1];
#else
// Will be shared and not copied.
UNREACHABLE();
#endif
}
void CopyTypedData(TypedDataPtr from, TypedDataPtr to) {
auto raw_from = from.untag();
auto raw_to = to.untag();
const intptr_t cid = Types::GetTypedDataPtr(from)->GetClassIdOfHeapObject();
raw_to->length_ = raw_from->length_;
raw_to->RecomputeDataField();
const intptr_t length =
TypedData::ElementSizeInBytes(cid) * Smi::Value(raw_from->length_);
memmove(raw_to->data_, raw_from->data_, length);
}
void CopyTypedData(const TypedData& from, const TypedData& to) {
auto raw_from = from.ptr().untag();
auto raw_to = to.ptr().untag();
const intptr_t cid = Types::GetTypedDataPtr(from)->GetClassIdOfHeapObject();
ASSERT(raw_to->length_ == raw_from->length_);
raw_to->RecomputeDataField();
const intptr_t length =
TypedData::ElementSizeInBytes(cid) * Smi::Value(raw_from->length_);
CopyTypedDataBaseWithSafepointChecks(Base::thread_, from, to, length);
}
void CopyTypedDataView(typename Types::TypedDataView from,
typename Types::TypedDataView to) {
// This will forward & initialize the typed data.
Base::ForwardCompressedPointer(
from, to, OFFSET_OF(UntaggedTypedDataView, typed_data_));
auto raw_from = UntagTypedDataView(from);
auto raw_to = UntagTypedDataView(to);
raw_to->length_ = raw_from->length_;
raw_to->offset_in_bytes_ = raw_from->offset_in_bytes_;
raw_to->data_ = nullptr;
auto forwarded_backing_store =
raw_to->typed_data_.Decompress(Base::heap_base_);
if (forwarded_backing_store == Marker() ||
forwarded_backing_store == Object::null()) {
// Ensure the backing store is never "sentinel" - the scavenger doesn't
// like it.
Base::StoreCompressedPointerNoBarrier(
Types::GetTypedDataViewPtr(to),
OFFSET_OF(UntaggedTypedDataView, typed_data_), Object::null());
raw_to->length_ = Smi::New(0);
raw_to->offset_in_bytes_ = Smi::New(0);
ASSERT(Base::exception_msg_ != nullptr);
return;
}
const bool is_external =
raw_from->data_ != raw_from->DataFieldForInternalTypedData();
if (is_external) {
// The raw_to is fully initialized at this point (see handling of external
// typed data in [ForwardCompressedPointer])
raw_to->RecomputeDataField();
} else {
// The raw_to isn't initialized yet, but it's address is valid, so we can
// compute the data field it would use.
raw_to->RecomputeDataFieldForInternalTypedData();
}
const bool is_external2 =
raw_to->data_ != raw_to->DataFieldForInternalTypedData();
ASSERT(is_external == is_external2);
}
void CopyExternalTypedData(typename Types::ExternalTypedData from,
typename Types::ExternalTypedData to) {
// The external typed data is initialized on the forwarding pass (where
// normally allocation but not initialization happens), so views on it
// can be initialized immediately.
#if defined(DEBUG)
auto raw_from = UntagExternalTypedData(from);
auto raw_to = UntagExternalTypedData(to);
ASSERT(raw_to->data_ != nullptr);
ASSERT(raw_to->length_ == raw_from->length_);
#endif
}
void CopyTransferableTypedData(typename Types::TransferableTypedData from,
typename Types::TransferableTypedData to) {
// The [TransferableTypedData] is an empty object with an associated heap
// peer object.
// -> We'll validate that there's a peer and enqueue the transferable to be
// transferred if the transitive copy is successful.
auto fpeer = static_cast<TransferableTypedDataPeer*>(
Base::heap_->GetPeer(Types::GetTransferableTypedDataPtr(from)));
ASSERT(fpeer != nullptr);
if (fpeer->data() == nullptr) {
Base::exception_msg_ =
"Illegal argument in isolate message"
" : (TransferableTypedData has been transferred already)";
Base::exception_unexpected_object_ =
Types::GetTransferableTypedDataPtr(from);
return;
}
Base::EnqueueTransferable(from, to);
}
void CopyWeakProperty(typename Types::WeakProperty from,
typename Types::WeakProperty to) {
// We store `null`s as keys/values and let the main algorithm know that
// we should check reachability of the key again after the fixpoint (if it
// became reachable, forward the key/value).
Base::StoreCompressedPointerNoBarrier(Types::GetWeakPropertyPtr(to),
OFFSET_OF(UntaggedWeakProperty, key_),
Object::null());
Base::StoreCompressedPointerNoBarrier(
Types::GetWeakPropertyPtr(to), OFFSET_OF(UntaggedWeakProperty, value_),
Object::null());
// To satisfy some ASSERT()s in GC we'll use Object:null() explicitly here.
Base::StoreCompressedPointerNoBarrier(
Types::GetWeakPropertyPtr(to),
OFFSET_OF(UntaggedWeakProperty, next_seen_by_gc_), Object::null());
Base::EnqueueWeakProperty(from);
}
void CopyWeakReference(typename Types::WeakReference from,
typename Types::WeakReference to) {
// We store `null` as target and let the main algorithm know that
// we should check reachability of the target again after the fixpoint (if
// it became reachable, forward the target).
Base::StoreCompressedPointerNoBarrier(
Types::GetWeakReferencePtr(to),
OFFSET_OF(UntaggedWeakReference, target_), Object::null());
// Type argument should always be copied.
Base::ForwardCompressedPointer(
from, to, OFFSET_OF(UntaggedWeakReference, type_arguments_));
// To satisfy some ASSERT()s in GC we'll use Object:null() explicitly here.
Base::StoreCompressedPointerNoBarrier(
Types::GetWeakReferencePtr(to),
OFFSET_OF(UntaggedWeakReference, next_seen_by_gc_), Object::null());
Base::EnqueueWeakReference(from);
}
// clang-format off
#define DEFINE_UNSUPPORTED(clazz) \
void Copy##clazz(typename Types::clazz from, typename Types::clazz to) { \
FATAL("Objects of type " #clazz " should not occur in object graphs"); \
}
FOR_UNSUPPORTED_CLASSES(DEFINE_UNSUPPORTED)
#undef DEFINE_UNSUPPORTED
// clang-format on
UntaggedObject* UntagObject(typename Types::Object obj) {
return Types::GetObjectPtr(obj).Decompress(Base::heap_base_).untag();
}
#define DO(V) \
DART_FORCE_INLINE \
Untagged##V* Untag##V(typename Types::V obj) { \
return Types::Get##V##Ptr(obj).Decompress(Base::heap_base_).untag(); \
}
CLASS_LIST_FOR_HANDLES(DO)
#undef DO
};
class FastObjectCopy : public ObjectCopy<FastObjectCopyBase> {
public:
FastObjectCopy(Thread* thread, IdentityMap* map) : ObjectCopy(thread, map) {}
~FastObjectCopy() {}
ObjectPtr TryCopyGraphFast(ObjectPtr root) {
NoSafepointScope no_safepoint_scope;
ObjectPtr root_copy = Forward(TagsFromUntaggedObject(root.untag()), root);
if (root_copy == Marker()) {
return root_copy;
}
auto& from_weak_property = WeakProperty::Handle(zone_);
auto& to_weak_property = WeakProperty::Handle(zone_);
auto& weak_property_key = Object::Handle(zone_);
while (true) {
if (fast_forward_map_.fill_cursor_ ==
fast_forward_map_.raw_from_to_.length()) {
break;
}
// Run fixpoint to copy all objects.
while (fast_forward_map_.fill_cursor_ <
fast_forward_map_.raw_from_to_.length()) {
const intptr_t index = fast_forward_map_.fill_cursor_;
ObjectPtr from = fast_forward_map_.raw_from_to_[index];
ObjectPtr to = fast_forward_map_.raw_from_to_[index + 1];
FastCopyObject(from, to);
if (exception_msg_ != nullptr) {
return root_copy;
}
fast_forward_map_.fill_cursor_ += 2;
// To maintain responsiveness we regularly check whether safepoints are
// requested - if so, we bail to slow path which will then checkin.
if (thread_->IsSafepointRequested()) {
exception_msg_ = kFastAllocationFailed;
return root_copy;
}
}
// Possibly forward values of [WeakProperty]s if keys became reachable.
intptr_t i = 0;
auto& weak_properties = fast_forward_map_.raw_weak_properties_;
while (i < weak_properties.length()) {
from_weak_property = weak_properties[i];
weak_property_key =
fast_forward_map_.ForwardedObject(from_weak_property.key());
if (weak_property_key.ptr() != Marker()) {
to_weak_property ^=
fast_forward_map_.ForwardedObject(from_weak_property.ptr());
// The key became reachable so we'll change the forwarded
// [WeakProperty]'s key to the new key (it is `null` at this point).
to_weak_property.set_key(weak_property_key);
// Since the key has become strongly reachable in the copied graph,
// we'll also need to forward the value.
ForwardCompressedPointer(from_weak_property.ptr(),
to_weak_property.ptr(),
OFFSET_OF(UntaggedWeakProperty, value_));
// We don't need to process this [WeakProperty] again.
const intptr_t last = weak_properties.length() - 1;
if (i < last) {
weak_properties[i] = weak_properties[last];
weak_properties.SetLength(last);
continue;
}
}
i++;
}
}
// After the fix point with [WeakProperty]s do [WeakReference]s.
auto& from_weak_reference = WeakReference::Handle(zone_);
auto& to_weak_reference = WeakReference::Handle(zone_);
auto& weak_reference_target = Object::Handle(zone_);
auto& weak_references = fast_forward_map_.raw_weak_references_;
for (intptr_t i = 0; i < weak_references.length(); i++) {
from_weak_reference = weak_references[i];
weak_reference_target =
fast_forward_map_.ForwardedObject(from_weak_reference.target());
if (weak_reference_target.ptr() != Marker()) {
to_weak_reference ^=
fast_forward_map_.ForwardedObject(from_weak_reference.ptr());
// The target became reachable so we'll change the forwarded
// [WeakReference]'s target to the new target (it is `null` at this
// point).
to_weak_reference.set_target(weak_reference_target);
}
}
if (root_copy != Marker()) {
ObjectPtr array;
array = TryBuildArrayOfObjectsToRehash(
fast_forward_map_.raw_objects_to_rehash_);
if (array == Marker()) return root_copy;
raw_objects_to_rehash_ = Array::RawCast(array);
array = TryBuildArrayOfObjectsToRehash(
fast_forward_map_.raw_expandos_to_rehash_);
if (array == Marker()) return root_copy;
raw_expandos_to_rehash_ = Array::RawCast(array);
}
return root_copy;
}
ObjectPtr TryBuildArrayOfObjectsToRehash(
const GrowableArray<ObjectPtr>& objects_to_rehash) {
const intptr_t length = objects_to_rehash.length();
if (length == 0) return Object::null();
const intptr_t size = Array::InstanceSize(length);
const uword array_addr = new_space_->TryAllocateNoSafepoint(thread_, size);
if (array_addr == 0) {
exception_msg_ = kFastAllocationFailed;
return Marker();
}
const uword header_size =
UntaggedObject::SizeTag::SizeFits(size) ? size : 0;
ArrayPtr array(reinterpret_cast<UntaggedArray*>(array_addr));
SetNewSpaceTaggingWord(array, kArrayCid, header_size);
StoreCompressedPointerNoBarrier(array, OFFSET_OF(UntaggedArray, length_),
Smi::New(length));
StoreCompressedPointerNoBarrier(array,
OFFSET_OF(UntaggedArray, type_arguments_),
TypeArguments::null());
auto array_data = array.untag()->data();
for (intptr_t i = 0; i < length; ++i) {
array_data[i] = objects_to_rehash[i];
}
return array;
}
private:
friend class ObjectGraphCopier;
void FastCopyObject(ObjectPtr from, ObjectPtr to) {
const uword tags = TagsFromUntaggedObject(from.untag());
const intptr_t cid = UntaggedObject::ClassIdTag::decode(tags);
const intptr_t size = UntaggedObject::SizeTag::decode(tags);
// Ensure the last word is GC-safe (our heap objects are 2-word aligned, the
// object header stores the size in multiples of kObjectAlignment, the GC
// uses the information from the header and therefore might visit one slot
// more than the actual size of the instance).
*reinterpret_cast<ObjectPtr*>(UntaggedObject::ToAddr(to) +
from.untag()->HeapSize() - kWordSize) =
nullptr;
SetNewSpaceTaggingWord(to, cid, size);
// Fall back to virtual variant for predefined classes
if (cid < kNumPredefinedCids && cid != kInstanceCid) {
CopyPredefinedInstance(from, to, cid);
return;
}
const auto bitmap = class_table_->GetUnboxedFieldsMapAt(cid);
CopyUserdefinedInstance(Instance::RawCast(from), Instance::RawCast(to),
bitmap);
if (cid == expando_cid_) {
EnqueueExpandoToRehash(to);
}
}
ArrayPtr raw_objects_to_rehash_ = Array::null();
ArrayPtr raw_expandos_to_rehash_ = Array::null();
};
class SlowObjectCopy : public ObjectCopy<SlowObjectCopyBase> {
public:
SlowObjectCopy(Thread* thread, IdentityMap* map)
: ObjectCopy(thread, map),
objects_to_rehash_(Array::Handle(thread->zone())),
expandos_to_rehash_(Array::Handle(thread->zone())) {}
~SlowObjectCopy() {}
ObjectPtr ContinueCopyGraphSlow(const Object& root,
const Object& fast_root_copy) {
auto& root_copy = Object::Handle(Z, fast_root_copy.ptr());
if (root_copy.ptr() == Marker()) {
root_copy = Forward(TagsFromUntaggedObject(root.ptr().untag()), root);
}
WeakProperty& weak_property = WeakProperty::Handle(Z);
Object& from = Object::Handle(Z);
Object& to = Object::Handle(Z);
while (true) {
if (slow_forward_map_.fill_cursor_ ==
slow_forward_map_.from_to_.Length()) {
break;
}
// Run fixpoint to copy all objects.
while (slow_forward_map_.fill_cursor_ <
slow_forward_map_.from_to_.Length()) {
const intptr_t index = slow_forward_map_.fill_cursor_;
from = slow_forward_map_.from_to_.At(index);
to = slow_forward_map_.from_to_.At(index + 1);
CopyObject(from, to);
slow_forward_map_.fill_cursor_ += 2;
if (exception_msg_ != nullptr) {
return Marker();
}
// To maintain responsiveness we regularly check whether safepoints are
// requested.
thread_->CheckForSafepoint();
}
// Possibly forward values of [WeakProperty]s if keys became reachable.
intptr_t i = 0;
auto& weak_properties = slow_forward_map_.weak_properties_;
while (i < weak_properties.length()) {
const auto& from_weak_property = *weak_properties[i];
to = slow_forward_map_.ForwardedObject(from_weak_property.key());
if (to.ptr() != Marker()) {
weak_property ^=
slow_forward_map_.ForwardedObject(from_weak_property.ptr());
// The key became reachable so we'll change the forwarded
// [WeakProperty]'s key to the new key (it is `null` at this point).
weak_property.set_key(to);
// Since the key has become strongly reachable in the copied graph,
// we'll also need to forward the value.
ForwardCompressedPointer(from_weak_property, weak_property,
OFFSET_OF(UntaggedWeakProperty, value_));
// We don't need to process this [WeakProperty] again.
const intptr_t last = weak_properties.length() - 1;
if (i < last) {
weak_properties[i] = weak_properties[last];
weak_properties.SetLength(last);
continue;
}
}
i++;
}
}
// After the fix point with [WeakProperty]s do [WeakReference]s.
WeakReference& weak_reference = WeakReference::Handle(Z);
auto& weak_references = slow_forward_map_.weak_references_;
for (intptr_t i = 0; i < weak_references.length(); i++) {
const auto& from_weak_reference = *weak_references[i];
to = slow_forward_map_.ForwardedObject(from_weak_reference.target());
if (to.ptr() != Marker()) {
weak_reference ^=
slow_forward_map_.ForwardedObject(from_weak_reference.ptr());
// The target became reachable so we'll change the forwarded
// [WeakReference]'s target to the new target (it is `null` at this
// point).
weak_reference.set_target(to);
}
}
objects_to_rehash_ =
BuildArrayOfObjectsToRehash(slow_forward_map_.objects_to_rehash_);
expandos_to_rehash_ =
BuildArrayOfObjectsToRehash(slow_forward_map_.expandos_to_rehash_);
return root_copy.ptr();
}
ArrayPtr BuildArrayOfObjectsToRehash(
const GrowableArray<const Object*>& objects_to_rehash) {
const intptr_t length = objects_to_rehash.length();
if (length == 0) return Array::null();
const auto& array = Array::Handle(zone_, Array::New(length));
for (intptr_t i = 0; i < length; ++i) {
array.SetAt(i, *objects_to_rehash[i]);
}
return array.ptr();
}
private:
friend class ObjectGraphCopier;
void CopyObject(const Object& from, const Object& to) {
const auto cid = from.GetClassId();
// Fall back to virtual variant for predefined classes
if (cid < kNumPredefinedCids && cid != kInstanceCid) {
CopyPredefinedInstance(from, to, cid);
return;
}
const auto bitmap = class_table_->GetUnboxedFieldsMapAt(cid);
CopyUserdefinedInstance(from, to, bitmap);
if (cid == expando_cid_) {
EnqueueExpandoToRehash(to);
}
}
Array& objects_to_rehash_;
Array& expandos_to_rehash_;
};
class ObjectGraphCopier : public StackResource {
public:
explicit ObjectGraphCopier(Thread* thread)
: StackResource(thread),
thread_(thread),
zone_(thread->zone()),
map_(thread),
fast_object_copy_(thread_, &map_),
slow_object_copy_(thread_, &map_) {}
// Result will be
// [
// <message>,
// <collection-lib-objects-to-rehash>,
// <core-lib-objects-to-rehash>,
// ]
ObjectPtr CopyObjectGraph(const Object& root) {
const char* volatile exception_msg = nullptr;
auto& result = Object::Handle(zone_);
{
LongJumpScope jump(thread_); // e.g. for OOMs.
if (DART_SETJMP(*jump.Set()) == 0) {
result = CopyObjectGraphInternal(root, &exception_msg);
// Any allocated external typed data must have finalizers attached so
// memory will get free()ed.
slow_object_copy_.slow_forward_map_.FinalizeExternalTypedData();
} else {
// Any allocated external typed data must have finalizers attached so
// memory will get free()ed.
slow_object_copy_.slow_forward_map_.FinalizeExternalTypedData();
// The copy failed due to non-application error (e.g. OOM error),
// propagate this error.
result = thread_->StealStickyError();
RELEASE_ASSERT(result.IsError());
}
}
if (result.IsError()) {
Exceptions::PropagateError(Error::Cast(result));
UNREACHABLE();
}
ASSERT(result.IsArray());
auto& result_array = Array::Cast(result);
if (result_array.At(0) == Marker()) {
ASSERT(exception_msg != nullptr);
auto& unexpected_object_ = Object::Handle(zone_, result_array.At(1));
if (!unexpected_object_.IsNull()) {
exception_msg = OS::SCreate(
zone_, "%s\n%s", exception_msg,
FindRetainingPath(zone_, thread_, root, unexpected_object_,
TraversalRules::kInternalToIsolateGroup));
}
ThrowException(exception_msg);
UNREACHABLE();
}
// The copy was successful, then detach transferable data from the sender
// and attach to the copied graph.
slow_object_copy_.slow_forward_map_.FinalizeTransferables();
return result.ptr();
}
intptr_t allocated_bytes() { return allocated_bytes_; }
intptr_t copied_objects() { return copied_objects_; }
private:
ObjectPtr CopyObjectGraphInternal(const Object& root,
const char* volatile* exception_msg) {
const auto& result_array = Array::Handle(zone_, Array::New(3));
if (!root.ptr()->IsHeapObject()) {
result_array.SetAt(0, root);
return result_array.ptr();
}
const uword tags = TagsFromUntaggedObject(root.ptr().untag());
if (CanShareObject(root.ptr(), tags)) {
result_array.SetAt(0, root);
return result_array.ptr();
}
if (!fast_object_copy_.CanCopyObject(tags, root.ptr())) {
ASSERT(fast_object_copy_.exception_msg_ != nullptr);
*exception_msg = fast_object_copy_.exception_msg_;
result_array.SetAt(0, Object::Handle(zone_, Marker()));
result_array.SetAt(1, fast_object_copy_.exception_unexpected_object_);
return result_array.ptr();
}
// We try a fast new-space only copy first that will not use any barriers.
auto& result = Object::Handle(Z, Marker());
// All allocated but non-initialized heap objects have to be made GC-visible
// at this point.
if (FLAG_enable_fast_object_copy) {
{
NoSafepointScope no_safepoint_scope;
result = fast_object_copy_.TryCopyGraphFast(root.ptr());
if (result.ptr() != Marker()) {
if (fast_object_copy_.exception_msg_ == nullptr) {
result_array.SetAt(0, result);
fast_object_copy_.tmp_ = fast_object_copy_.raw_objects_to_rehash_;
result_array.SetAt(1, fast_object_copy_.tmp_);
fast_object_copy_.tmp_ = fast_object_copy_.raw_expandos_to_rehash_;
result_array.SetAt(2, fast_object_copy_.tmp_);
HandlifyExternalTypedData();
HandlifyTransferables();
allocated_bytes_ =
fast_object_copy_.fast_forward_map_.allocated_bytes;
copied_objects_ =
fast_object_copy_.fast_forward_map_.fill_cursor_ / 2 -
/*null_entry=*/1;
return result_array.ptr();
}
// There are left-over uninitialized objects we'll have to make GC
// visible.
SwitchToSlowForwardingList();
}
}
if (FLAG_gc_on_foc_slow_path) {
// We force the GC to compact, which is more likely to discover
// untracked pointers (and other issues, like incorrect class table).
thread_->heap()->CollectAllGarbage(GCReason::kDebugging,
/*compact=*/true);
}
ObjectifyFromToObjects();
// Fast copy failed due to
// - either failure to allocate into new space
// - or failure to copy object which we cannot copy
ASSERT(fast_object_copy_.exception_msg_ != nullptr);
if (fast_object_copy_.exception_msg_ != kFastAllocationFailed) {
*exception_msg = fast_object_copy_.exception_msg_;
result_array.SetAt(0, Object::Handle(zone_, Marker()));
result_array.SetAt(1, fast_object_copy_.exception_unexpected_object_);
return result_array.ptr();
}
ASSERT(fast_object_copy_.exception_msg_ == kFastAllocationFailed);
}
// Use the slow copy approach.
result = slow_object_copy_.ContinueCopyGraphSlow(root, result);
ASSERT((result.ptr() == Marker()) ==
(slow_object_copy_.exception_msg_ != nullptr));
if (result.ptr() == Marker()) {
*exception_msg = slow_object_copy_.exception_msg_;
result_array.SetAt(0, Object::Handle(zone_, Marker()));
result_array.SetAt(1, slow_object_copy_.exception_unexpected_object_);
return result_array.ptr();
}
result_array.SetAt(0, result);
result_array.SetAt(1, slow_object_copy_.objects_to_rehash_);
result_array.SetAt(2, slow_object_copy_.expandos_to_rehash_);
allocated_bytes_ = slow_object_copy_.slow_forward_map_.allocated_bytes;
copied_objects_ =
slow_object_copy_.slow_forward_map_.fill_cursor_ / 2 - /*null_entry=*/1;
return result_array.ptr();
}
void SwitchToSlowForwardingList() {
auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
auto& slow_forward_map = slow_object_copy_.slow_forward_map_;
MakeUninitializedNewSpaceObjectsGCSafe();
HandlifyTransferables();
HandlifyWeakProperties();
HandlifyWeakReferences();
HandlifyExternalTypedData();
HandlifyObjectsToReHash();
HandlifyExpandosToReHash();
HandlifyFromToObjects();
slow_forward_map.fill_cursor_ = fast_forward_map.fill_cursor_;
slow_forward_map.allocated_bytes = fast_forward_map.allocated_bytes;
}
void MakeUninitializedNewSpaceObjectsGCSafe() {
auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
const auto length = fast_forward_map.raw_from_to_.length();
const auto cursor = fast_forward_map.fill_cursor_;
for (intptr_t i = cursor; i < length; i += 2) {
auto from = fast_forward_map.raw_from_to_[i];
auto to = fast_forward_map.raw_from_to_[i + 1];
const uword tags = TagsFromUntaggedObject(from.untag());
const intptr_t cid = UntaggedObject::ClassIdTag::decode(tags);
// External typed data is already initialized.
if (!IsExternalTypedDataClassId(cid) && !IsTypedDataViewClassId(cid) &&
!IsUnmodifiableTypedDataViewClassId(cid)) {
#if defined(DART_COMPRESSED_POINTERS)
const bool compressed = true;
#else
const bool compressed = false;
#endif
// Mimic the old initialization behavior of Object::InitializeObject
// where the contents are initialized to Object::null(), except for
// TypedDataBase subclasses which are initialized to 0, as the contents
// of the original are translated and copied over prior to returning
// the object graph root.
if (IsTypedDataBaseClassId(cid)) {
Object::InitializeObject(reinterpret_cast<uword>(to.untag()), cid,
from.untag()->HeapSize(), compressed,
Object::from_offset<TypedDataBase>(),
Object::to_offset<TypedDataBase>());
} else {
// Remember that ptr_field_end_offset is the offset to the last Ptr
// field, not the offset just past it.
const uword ptr_field_end_offset =
from.untag()->HeapSize() -
(compressed ? kCompressedWordSize : kWordSize);
Object::InitializeObject(reinterpret_cast<uword>(to.untag()), cid,
from.untag()->HeapSize(), compressed,
Object::from_offset<Object>(),
ptr_field_end_offset);
}
UpdateLengthField(cid, from, to);
}
}
}
void HandlifyTransferables() {
Handlify(&fast_object_copy_.fast_forward_map_.raw_transferables_from_to_,
&slow_object_copy_.slow_forward_map_.transferables_from_to_);
}
void HandlifyWeakProperties() {
Handlify(&fast_object_copy_.fast_forward_map_.raw_weak_properties_,
&slow_object_copy_.slow_forward_map_.weak_properties_);
}
void HandlifyWeakReferences() {
Handlify(&fast_object_copy_.fast_forward_map_.raw_weak_references_,
&slow_object_copy_.slow_forward_map_.weak_references_);
}
void HandlifyExternalTypedData() {
Handlify(&fast_object_copy_.fast_forward_map_.raw_external_typed_data_to_,
&slow_object_copy_.slow_forward_map_.external_typed_data_);
}
void HandlifyObjectsToReHash() {
Handlify(&fast_object_copy_.fast_forward_map_.raw_objects_to_rehash_,
&slow_object_copy_.slow_forward_map_.objects_to_rehash_);
}
void HandlifyExpandosToReHash() {
Handlify(&fast_object_copy_.fast_forward_map_.raw_expandos_to_rehash_,
&slow_object_copy_.slow_forward_map_.expandos_to_rehash_);
}
template <typename PtrType, typename HandleType>
void Handlify(GrowableArray<PtrType>* from,
GrowableArray<const HandleType*>* to) {
const auto length = from->length();
if (length > 0) {
to->Resize(length);
for (intptr_t i = 0; i < length; i++) {
(*to)[i] = &HandleType::Handle(Z, (*from)[i]);
}
from->Clear();
}
}
void HandlifyFromToObjects() {
auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
auto& slow_forward_map = slow_object_copy_.slow_forward_map_;
const intptr_t length = fast_forward_map.raw_from_to_.length();
slow_forward_map.from_to_transition_.Resize(length);
for (intptr_t i = 0; i < length; i++) {
slow_forward_map.from_to_transition_[i] =
&PassiveObject::Handle(Z, fast_forward_map.raw_from_to_[i]);
}
ASSERT(slow_forward_map.from_to_transition_.length() == length);
fast_forward_map.raw_from_to_.Clear();
}
void ObjectifyFromToObjects() {
auto& from_to_transition =
slow_object_copy_.slow_forward_map_.from_to_transition_;
auto& from_to = slow_object_copy_.slow_forward_map_.from_to_;
intptr_t length = from_to_transition.length();
from_to = GrowableObjectArray::New(length, Heap::kOld);
for (intptr_t i = 0; i < length; i++) {
from_to.Add(*from_to_transition[i]);
}
ASSERT(from_to.Length() == length);
from_to_transition.Clear();
}
void ThrowException(const char* exception_msg) {
const auto& msg_obj = String::Handle(Z, String::New(exception_msg));
const auto& args = Array::Handle(Z, Array::New(1));
args.SetAt(0, msg_obj);
Exceptions::ThrowByType(Exceptions::kArgument, args);
UNREACHABLE();
}
Thread* thread_;
Zone* zone_;
IdentityMap map_;
FastObjectCopy fast_object_copy_;
SlowObjectCopy slow_object_copy_;
intptr_t copied_objects_ = 0;
intptr_t allocated_bytes_ = 0;
};
ObjectPtr CopyMutableObjectGraph(const Object& object) {
auto thread = Thread::Current();
TIMELINE_DURATION(thread, Isolate, "CopyMutableObjectGraph");
ObjectGraphCopier copier(thread);
ObjectPtr result = copier.CopyObjectGraph(object);
#if defined(SUPPORT_TIMELINE)
if (tbes.enabled()) {
tbes.SetNumArguments(2);
tbes.FormatArgument(0, "CopiedObjects", "%" Pd, copier.copied_objects());
tbes.FormatArgument(1, "AllocatedBytes", "%" Pd, copier.allocated_bytes());
}
#endif
return result;
}
} // namespace dart