[vm, gc] Common page structure for both generations. - Prep work for non-moving promotion. - The generations still have different alignment offsets. - Old-space pages now participate in the page cache. - Fix boolean/null bit tricks to assert the right requirements on the alignment of a page's first object. TEST=ci Change-Id: I4369d8c6af73228e162c226d411914868bafed33 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/260401 Reviewed-by: Siva Annamalai <asiva@google.com> Commit-Queue: Ryan Macnak <rmacnak@google.com>
diff --git a/runtime/vm/compiler/relocation.cc b/runtime/vm/compiler/relocation.cc index 08641a0..f17ae04 100644 --- a/runtime/vm/compiler/relocation.cc +++ b/runtime/vm/compiler/relocation.cc
@@ -344,7 +344,7 @@ auto const caller = unresolved_call->caller; uword addr = Code::PayloadStartOf(caller) + call_offset; if (FLAG_write_protect_code) { - addr -= OldPage::Of(Code::InstructionsOf(caller))->AliasOffset(); + addr -= Page::Of(Code::InstructionsOf(caller))->AliasOffset(); } if (unresolved_call->is_tail_call) { PcRelativeTailCallPattern call(addr);
diff --git a/runtime/vm/compiler/relocation_test.cc b/runtime/vm/compiler/relocation_test.cc index 4aec712..ebced27 100644 --- a/runtime/vm/compiler/relocation_test.cc +++ b/runtime/vm/compiler/relocation_test.cc
@@ -136,7 +136,7 @@ if (FLAG_write_protect_code && FLAG_dual_map_code) { auto& instructions = Instructions::Handle(code.instructions()); - instructions ^= OldPage::ToExecutable(instructions.ptr()); + instructions ^= Page::ToExecutable(instructions.ptr()); code.set_instructions(instructions); } if (FLAG_disassemble) { @@ -245,7 +245,7 @@ const auto current_size = ImageWriter::SizeInSnapshot(Code::InstructionsOf(entry.code)); const auto alias_offset = - OldPage::Of(Code::InstructionsOf(entry.code))->AliasOffset(); + Page::Of(Code::InstructionsOf(entry.code))->AliasOffset(); memmove( reinterpret_cast<void*>(addr), reinterpret_cast<void*>(Instructions::PayloadStart( @@ -262,7 +262,7 @@ const uword address = UntaggedObject::ToAddr(instructions.ptr()); const auto size = instructions.ptr()->untag()->HeapSize(); instructions = - Instructions::RawCast(OldPage::ToExecutable(instructions.ptr())); + Instructions::RawCast(Page::ToExecutable(instructions.ptr())); const auto prot = FLAG_dual_map_code ? VirtualMemory::kReadOnly : VirtualMemory::kReadExecute;
diff --git a/runtime/vm/compiler/runtime_api.cc b/runtime/vm/compiler/runtime_api.cc index 5e72ade..0ae58f0 100644 --- a/runtime/vm/compiler/runtime_api.cc +++ b/runtime/vm/compiler/runtime_api.cc
@@ -328,9 +328,9 @@ namespace target { -const word kOldPageSize = dart::kOldPageSize; -const word kOldPageSizeInWords = dart::kOldPageSize / kWordSize; -const word kOldPageMask = dart::kOldPageMask; +const word kPageSize = dart::kPageSize; +const word kPageSizeInWords = dart::kPageSize / kWordSize; +const word kPageMask = dart::kPageMask; static word TranslateOffsetInWordsToHost(word offset) { RELEASE_ASSERT((offset % kCompressedWordSize) == 0);
diff --git a/runtime/vm/compiler/runtime_api.h b/runtime/vm/compiler/runtime_api.h index 86be807..7afa8c7 100644 --- a/runtime/vm/compiler/runtime_api.h +++ b/runtime/vm/compiler/runtime_api.h
@@ -77,9 +77,9 @@ extern InvalidClass kNewObjectAlignmentOffset; extern InvalidClass kOldObjectAlignmentOffset; extern InvalidClass kNewObjectBitPosition; -extern InvalidClass kOldPageSize; -extern InvalidClass kOldPageSizeInWords; -extern InvalidClass kOldPageMask; +extern InvalidClass kPageSize; +extern InvalidClass kPageSizeInWords; +extern InvalidClass kPageMask; extern InvalidClass kObjectAlignment; extern InvalidClass kObjectAlignmentLog2; extern InvalidClass kObjectAlignmentMask; @@ -301,9 +301,9 @@ constexpr word kSmiMin = -(static_cast<uword>(1) << kSmiBits); // Information about heap pages. -extern const word kOldPageSize; -extern const word kOldPageSizeInWords; -extern const word kOldPageMask; +extern const word kPageSize; +extern const word kPageSizeInWords; +extern const word kPageMask; static constexpr intptr_t kObjectAlignment = ObjectAlignment::kObjectAlignment; @@ -1462,7 +1462,7 @@ FINAL_CLASS(); }; -class OldPage : public AllStatic { +class Page : public AllStatic { public: static const word kBytesPerCardLog2;
diff --git a/runtime/vm/compiler/runtime_offsets_extracted.h b/runtime/vm/compiler/runtime_offsets_extracted.h index 8100475..6935a83 100644 --- a/runtime/vm/compiler/runtime_offsets_extracted.h +++ b/runtime/vm/compiler/runtime_offsets_extracted.h
@@ -73,7 +73,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 4; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = 536870911; @@ -176,7 +176,7 @@ GrowableObjectArray_length_offset = 8; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 4; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 20; +static constexpr dart::compiler::target::word Page_card_table_offset = 16; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 8; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -733,7 +733,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 10; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 10; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = @@ -837,7 +837,7 @@ GrowableObjectArray_length_offset = 16; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 40; +static constexpr dart::compiler::target::word Page_card_table_offset = 32; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -1400,7 +1400,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 4; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = 536870911; @@ -1503,7 +1503,7 @@ GrowableObjectArray_length_offset = 8; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 4; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 20; +static constexpr dart::compiler::target::word Page_card_table_offset = 16; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 8; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -2057,7 +2057,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 10; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 10; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = @@ -2161,7 +2161,7 @@ GrowableObjectArray_length_offset = 16; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 40; +static constexpr dart::compiler::target::word Page_card_table_offset = 32; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -2725,7 +2725,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = 536870911; @@ -2828,7 +2828,7 @@ GrowableObjectArray_length_offset = 12; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 40; +static constexpr dart::compiler::target::word Page_card_table_offset = 32; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -3391,7 +3391,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = 536870911; @@ -3494,7 +3494,7 @@ GrowableObjectArray_length_offset = 12; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 40; +static constexpr dart::compiler::target::word Page_card_table_offset = 32; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -4058,7 +4058,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 4; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = 536870911; @@ -4161,7 +4161,7 @@ GrowableObjectArray_length_offset = 8; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 4; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 20; +static constexpr dart::compiler::target::word Page_card_table_offset = 16; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 8; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -4720,7 +4720,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 10; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 10; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = @@ -4824,7 +4824,7 @@ GrowableObjectArray_length_offset = 16; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 40; +static constexpr dart::compiler::target::word Page_card_table_offset = 32; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -5387,7 +5387,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 4; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = 536870911; @@ -5488,7 +5488,7 @@ GrowableObjectArray_length_offset = 8; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 4; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 20; +static constexpr dart::compiler::target::word Page_card_table_offset = 16; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 8; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -6039,7 +6039,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 10; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 10; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = @@ -6141,7 +6141,7 @@ GrowableObjectArray_length_offset = 16; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 40; +static constexpr dart::compiler::target::word Page_card_table_offset = 32; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -6698,7 +6698,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 4; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = 536870911; @@ -6799,7 +6799,7 @@ GrowableObjectArray_length_offset = 8; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 4; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 20; +static constexpr dart::compiler::target::word Page_card_table_offset = 16; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 8; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -7347,7 +7347,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 10; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 10; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = @@ -7449,7 +7449,7 @@ GrowableObjectArray_length_offset = 16; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 40; +static constexpr dart::compiler::target::word Page_card_table_offset = 32; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -8007,7 +8007,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = 536870911; @@ -8108,7 +8108,7 @@ GrowableObjectArray_length_offset = 12; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 40; +static constexpr dart::compiler::target::word Page_card_table_offset = 32; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -8665,7 +8665,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = 536870911; @@ -8766,7 +8766,7 @@ GrowableObjectArray_length_offset = 12; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 40; +static constexpr dart::compiler::target::word Page_card_table_offset = 32; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -9324,7 +9324,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 4; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = 536870911; @@ -9425,7 +9425,7 @@ GrowableObjectArray_length_offset = 8; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 4; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 20; +static constexpr dart::compiler::target::word Page_card_table_offset = 16; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 8; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -9978,7 +9978,7 @@ Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 10; +static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 10; static constexpr dart::compiler::target::word NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word String_kMaxElements = @@ -10080,7 +10080,7 @@ GrowableObjectArray_length_offset = 16; static constexpr dart::compiler::target::word GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word OldPage_card_table_offset = 40; +static constexpr dart::compiler::target::word Page_card_table_offset = 32; static constexpr dart::compiler::target::word CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3; @@ -10648,7 +10648,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 4; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -10756,8 +10756,7 @@ AOT_GrowableObjectArray_length_offset = 8; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 4; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 20; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 16; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 8; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3; @@ -11382,8 +11381,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = - 10; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 10; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -11492,8 +11490,7 @@ AOT_GrowableObjectArray_length_offset = 16; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 40; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3; @@ -12125,8 +12122,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = - 10; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 10; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -12235,8 +12231,7 @@ AOT_GrowableObjectArray_length_offset = 16; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 40; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3; @@ -12866,7 +12861,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -12975,8 +12970,7 @@ AOT_GrowableObjectArray_length_offset = 12; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 40; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3; @@ -13605,7 +13599,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -13714,8 +13708,7 @@ AOT_GrowableObjectArray_length_offset = 12; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 40; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3; @@ -14345,7 +14338,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 4; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -14453,8 +14446,7 @@ AOT_GrowableObjectArray_length_offset = 8; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 4; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 20; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 16; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 8; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3; @@ -15081,8 +15073,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = - 10; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 10; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -15191,8 +15182,7 @@ AOT_GrowableObjectArray_length_offset = 16; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 40; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3; @@ -15821,7 +15811,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 4; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -15927,8 +15917,7 @@ AOT_GrowableObjectArray_length_offset = 8; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 4; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 20; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 16; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 8; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3; @@ -16546,8 +16535,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = - 10; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 10; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -16654,8 +16642,7 @@ AOT_GrowableObjectArray_length_offset = 16; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 40; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3; @@ -17280,8 +17267,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = - 10; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 10; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -17388,8 +17374,7 @@ AOT_GrowableObjectArray_length_offset = 16; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 40; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3; @@ -18012,7 +17997,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -18119,8 +18104,7 @@ AOT_GrowableObjectArray_length_offset = 12; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 40; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3; @@ -18742,7 +18726,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -18849,8 +18833,7 @@ AOT_GrowableObjectArray_length_offset = 12; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 40; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3; @@ -19473,7 +19456,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 4; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -19579,8 +19562,7 @@ AOT_GrowableObjectArray_length_offset = 8; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 4; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 20; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 16; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 8; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3; @@ -20200,8 +20182,7 @@ AOT_Instructions_kBarePayloadAlignment = 4; static constexpr dart::compiler::target::word AOT_Instructions_kNonBarePayloadAlignment = 8; -static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = - 10; +static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 10; static constexpr dart::compiler::target::word AOT_NativeEntry_kNumCallWrapperArguments = 2; static constexpr dart::compiler::target::word AOT_String_kMaxElements = @@ -20308,8 +20289,7 @@ AOT_GrowableObjectArray_length_offset = 16; static constexpr dart::compiler::target::word AOT_GrowableObjectArray_type_arguments_offset = 8; -static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset = - 40; +static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32; static constexpr dart::compiler::target::word AOT_CallSiteData_arguments_descriptor_offset = 16; static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
diff --git a/runtime/vm/compiler/runtime_offsets_list.h b/runtime/vm/compiler/runtime_offsets_list.h index 4ed9792..a023962b 100644 --- a/runtime/vm/compiler/runtime_offsets_list.h +++ b/runtime/vm/compiler/runtime_offsets_list.h
@@ -75,7 +75,7 @@ CONSTANT(Instructions, kPolymorphicEntryOffsetAOT) \ CONSTANT(Instructions, kBarePayloadAlignment) \ CONSTANT(Instructions, kNonBarePayloadAlignment) \ - CONSTANT(OldPage, kBytesPerCardLog2) \ + CONSTANT(Page, kBytesPerCardLog2) \ CONSTANT(NativeEntry, kNumCallWrapperArguments) \ CONSTANT(String, kMaxElements) \ CONSTANT(SubtypeTestCache, kFunctionTypeArguments) \ @@ -141,7 +141,7 @@ FIELD(GrowableObjectArray, data_offset) \ FIELD(GrowableObjectArray, length_offset) \ FIELD(GrowableObjectArray, type_arguments_offset) \ - FIELD(OldPage, card_table_offset) \ + FIELD(Page, card_table_offset) \ FIELD(CallSiteData, arguments_descriptor_offset) \ FIELD(ICData, NumArgsTestedMask) \ FIELD(ICData, NumArgsTestedShift) \
diff --git a/runtime/vm/compiler/stub_code_compiler_arm.cc b/runtime/vm/compiler/stub_code_compiler_arm.cc index e50effc..7e26f2c 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm.cc
@@ -1719,20 +1719,20 @@ // Get card table. __ Bind(&remember_card); - __ AndImmediate(TMP, R1, target::kOldPageMask); // OldPage. + __ AndImmediate(TMP, R1, target::kPageMask); // Page. __ ldr(TMP, - Address(TMP, target::OldPage::card_table_offset())); // Card table. + Address(TMP, target::Page::card_table_offset())); // Card table. __ cmp(TMP, Operand(0)); __ b(&remember_card_slow, EQ); // Dirty the card. - __ AndImmediate(TMP, R1, target::kOldPageMask); // OldPage. + __ AndImmediate(TMP, R1, target::kPageMask); // Page. __ sub(R9, R9, Operand(TMP)); // Offset in page. __ ldr(TMP, - Address(TMP, target::OldPage::card_table_offset())); // Card table. + Address(TMP, target::Page::card_table_offset())); // Card table. __ add(TMP, TMP, Operand(R9, LSR, - target::OldPage::kBytesPerCardLog2)); // Card address. + target::Page::kBytesPerCardLog2)); // Card address. __ strb(R1, Address(TMP, 0)); // Low byte of R0 is non-zero from object tag. __ Ret();
diff --git a/runtime/vm/compiler/stub_code_compiler_arm64.cc b/runtime/vm/compiler/stub_code_compiler_arm64.cc index 40c1d94..1658a12 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm64.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm64.cc
@@ -2024,19 +2024,19 @@ // Get card table. __ Bind(&remember_card); - __ AndImmediate(TMP, R1, target::kOldPageMask); // OldPage. + __ AndImmediate(TMP, R1, target::kPageMask); // Page. __ ldr(TMP, - Address(TMP, target::OldPage::card_table_offset())); // Card table. + Address(TMP, target::Page::card_table_offset())); // Card table. __ cbz(&remember_card_slow, TMP); // Dirty the card. - __ AndImmediate(TMP, R1, target::kOldPageMask); // OldPage. + __ AndImmediate(TMP, R1, target::kPageMask); // Page. __ sub(R25, R25, Operand(TMP)); // Offset in page. __ ldr(TMP, - Address(TMP, target::OldPage::card_table_offset())); // Card table. + Address(TMP, target::Page::card_table_offset())); // Card table. __ add(TMP, TMP, Operand(R25, LSR, - target::OldPage::kBytesPerCardLog2)); // Card address. + target::Page::kBytesPerCardLog2)); // Card address. __ str(R1, Address(TMP, 0), kUnsignedByte); // Low byte of R1 is non-zero from object tag. __ ret();
diff --git a/runtime/vm/compiler/stub_code_compiler_ia32.cc b/runtime/vm/compiler/stub_code_compiler_ia32.cc index d1ffea3..cb0b674 100644 --- a/runtime/vm/compiler/stub_code_compiler_ia32.cc +++ b/runtime/vm/compiler/stub_code_compiler_ia32.cc
@@ -1454,17 +1454,17 @@ // Get card table. __ Bind(&remember_card); __ movl(EAX, EDX); // Object. - __ andl(EAX, Immediate(target::kOldPageMask)); // OldPage. - __ cmpl(Address(EAX, target::OldPage::card_table_offset()), Immediate(0)); + __ andl(EAX, Immediate(target::kPageMask)); // Page. + __ cmpl(Address(EAX, target::Page::card_table_offset()), Immediate(0)); __ j(EQUAL, &remember_card_slow, Assembler::kNearJump); // Dirty the card. __ subl(EDI, EAX); // Offset in page. __ movl(EAX, - Address(EAX, target::OldPage::card_table_offset())); // Card table. + Address(EAX, target::Page::card_table_offset())); // Card table. __ shrl( EDI, - Immediate(target::OldPage::kBytesPerCardLog2)); // Index in card table. + Immediate(target::Page::kBytesPerCardLog2)); // Index in card table. __ movb(Address(EAX, EDI, TIMES_1, 0), Immediate(1)); __ popl(ECX); __ popl(EAX);
diff --git a/runtime/vm/compiler/stub_code_compiler_riscv.cc b/runtime/vm/compiler/stub_code_compiler_riscv.cc index d281e7b..57add7e 100644 --- a/runtime/vm/compiler/stub_code_compiler_riscv.cc +++ b/runtime/vm/compiler/stub_code_compiler_riscv.cc
@@ -1842,17 +1842,15 @@ // Get card table. __ Bind(&remember_card); - __ AndImmediate(TMP, A0, target::kOldPageMask); // OldPage. - __ lx(TMP, - Address(TMP, target::OldPage::card_table_offset())); // Card table. + __ AndImmediate(TMP, A0, target::kPageMask); // Page. + __ lx(TMP, Address(TMP, target::Page::card_table_offset())); // Card table. __ beqz(TMP, &remember_card_slow); // Dirty the card. - __ AndImmediate(TMP, A0, target::kOldPageMask); // OldPage. + __ AndImmediate(TMP, A0, target::kPageMask); // Page. __ sub(A6, A6, TMP); // Offset in page. - __ lx(TMP, - Address(TMP, target::OldPage::card_table_offset())); // Card table. - __ srli(A6, A6, target::OldPage::kBytesPerCardLog2); + __ lx(TMP, Address(TMP, target::Page::card_table_offset())); // Card table. + __ srli(A6, A6, target::Page::kBytesPerCardLog2); __ add(TMP, TMP, A6); // Card address. __ sb(A0, Address(TMP, 0)); // Low byte of A0 is non-zero from object tag. __ ret();
diff --git a/runtime/vm/compiler/stub_code_compiler_x64.cc b/runtime/vm/compiler/stub_code_compiler_x64.cc index 26dba7f..c2cb5e1 100644 --- a/runtime/vm/compiler/stub_code_compiler_x64.cc +++ b/runtime/vm/compiler/stub_code_compiler_x64.cc
@@ -1928,17 +1928,17 @@ // Get card table. __ Bind(&remember_card); __ movq(TMP, RDX); // Object. - __ andq(TMP, Immediate(target::kOldPageMask)); // OldPage. - __ cmpq(Address(TMP, target::OldPage::card_table_offset()), Immediate(0)); + __ andq(TMP, Immediate(target::kPageMask)); // Page. + __ cmpq(Address(TMP, target::Page::card_table_offset()), Immediate(0)); __ j(EQUAL, &remember_card_slow, Assembler::kNearJump); // Dirty the card. __ subq(R13, TMP); // Offset in page. __ movq(TMP, - Address(TMP, target::OldPage::card_table_offset())); // Card table. + Address(TMP, target::Page::card_table_offset())); // Card table. __ shrq( R13, - Immediate(target::OldPage::kBytesPerCardLog2)); // Index in card table. + Immediate(target::Page::kBytesPerCardLog2)); // Index in card table. __ movb(Address(TMP, R13, TIMES_1, 0), Immediate(1)); __ ret();
diff --git a/runtime/vm/dart.cc b/runtime/vm/dart.cc index 9683ee1..a0e900a8 100644 --- a/runtime/vm/dart.cc +++ b/runtime/vm/dart.cc
@@ -326,7 +326,7 @@ Api::Init(); NativeSymbolResolver::Init(); NOT_IN_PRODUCT(Profiler::Init()); - SemiSpace::Init(); + Page::Init(); NOT_IN_PRODUCT(Metric::Init()); StoreBuffer::Init(); MarkingStack::Init(); @@ -773,7 +773,7 @@ MarkingStack::Cleanup(); StoreBuffer::Cleanup(); Object::Cleanup(); - SemiSpace::Cleanup(); + Page::Cleanup(); StubCode::Cleanup(); #if defined(SUPPORT_TIMELINE) if (FLAG_trace_shutdown) {
diff --git a/runtime/vm/dart_api_impl.cc b/runtime/vm/dart_api_impl.cc index 30a3631..600f3f0 100644 --- a/runtime/vm/dart_api_impl.cc +++ b/runtime/vm/dart_api_impl.cc
@@ -1820,7 +1820,7 @@ DART_EXPORT void Dart_NotifyLowMemory() { API_TIMELINE_BEGIN_END(Thread::Current()); - SemiSpace::ClearCache(); + Page::ClearCache(); Zone::ClearCache(); // For each isolate's global variables, we might also clear:
diff --git a/runtime/vm/globals.h b/runtime/vm/globals.h index ad5687b..80b830e 100644 --- a/runtime/vm/globals.h +++ b/runtime/vm/globals.h
@@ -56,7 +56,7 @@ // The default old gen heap size in MB, where 0 -- unlimited. // 32-bit: OS limit is 2 or 3 GB // 64-bit: Linux's limit is -// sysctl vm.max_map_count (default 2^16) * 512 KB OldPages = 32 GB +// sysctl vm.max_map_count (default 2^16) * 512 KB Pages = 32 GB // Set the VM limit below the OS limit to increase the likelihood of failing // gracefully with a Dart OutOfMemory exception instead of SIGABORT. const intptr_t kDefaultMaxOldGenHeapSize = (kWordSize <= 4) ? 1536 : 30720;
diff --git a/runtime/vm/heap/compactor.cc b/runtime/vm/heap/compactor.cc index 9afb2d7..7a3ca54 100644 --- a/runtime/vm/heap/compactor.cc +++ b/runtime/vm/heap/compactor.cc
@@ -19,11 +19,11 @@ false, "Force compaction to move every movable object"); -// Each OldPage is divided into blocks of size kBlockSize. Each object belongs +// Each Page is divided into blocks of size kBlockSize. Each object belongs // to the block containing its header word (so up to kBlockSize + // kAllocatablePageSize - 2 * kObjectAlignment bytes belong to the same block). // During compaction, all live objects in the same block will slide such that -// they all end up on the same OldPage, and all gaps within the block will be +// they all end up on the same Page, and all gaps within the block will be // closed. During sliding, a bitvector is computed that indictates which // allocation units are live, so the new address of any object in the block can // be found by adding the number of live allocation units before the object to @@ -95,7 +95,7 @@ uword Lookup(uword old_addr) { return BlockFor(old_addr)->Lookup(old_addr); } ForwardingBlock* BlockFor(uword old_addr) { - intptr_t page_offset = old_addr & ~kOldPageMask; + intptr_t page_offset = old_addr & ~kPageMask; intptr_t block_number = page_offset / kBlockSize; ASSERT(block_number >= 0); ASSERT(block_number <= kBlocksPerPage); @@ -109,17 +109,17 @@ DISALLOW_IMPLICIT_CONSTRUCTORS(ForwardingPage); }; -void OldPage::AllocateForwardingPage() { +void Page::AllocateForwardingPage() { ASSERT(forwarding_page_ == NULL); ASSERT((object_start() + sizeof(ForwardingPage)) < object_end()); ASSERT(Utils::IsAligned(sizeof(ForwardingPage), kObjectAlignment)); - object_end_ -= sizeof(ForwardingPage); - forwarding_page_ = reinterpret_cast<ForwardingPage*>(object_end_); + top_ -= sizeof(ForwardingPage); + forwarding_page_ = reinterpret_cast<ForwardingPage*>(top_); } struct Partition { - OldPage* head; - OldPage* tail; + Page* head; + Page* tail; }; class CompactorTask : public ThreadPool::Task { @@ -150,8 +150,8 @@ void RunEnteredIsolateGroup(); private: - void PlanPage(OldPage* page); - void SlidePage(OldPage* page); + void PlanPage(Page* page); + void SlidePage(Page* page); uword PlanBlock(uword first_object, ForwardingPage* forwarding_page); uword SlideBlock(uword first_object, ForwardingPage* forwarding_page); void PlanMoveToContiguousSize(intptr_t size); @@ -165,7 +165,7 @@ intptr_t num_tasks_; Partition* partitions_; FreeList* freelist_; - OldPage* free_page_; + Page* free_page_; uword free_current_; uword free_end_; @@ -178,15 +178,13 @@ // time, keeping blocks from spanning page boundaries (see ForwardingBlock). // Free space at the end of a page that is too small for the next block is // added to the freelist. -void GCCompactor::Compact(OldPage* pages, - FreeList* freelist, - Mutex* pages_lock) { +void GCCompactor::Compact(Page* pages, FreeList* freelist, Mutex* pages_lock) { SetupImagePageBoundaries(); // Divide the heap. // TODO(30978): Try to divide based on live bytes or with work stealing. intptr_t num_pages = 0; - for (OldPage* page = pages; page != NULL; page = page->next()) { + for (Page* page = pages; page != NULL; page = page->next()) { num_pages++; } @@ -202,8 +200,8 @@ const intptr_t pages_per_task = num_pages / num_tasks; intptr_t task_index = 0; intptr_t page_index = 0; - OldPage* page = pages; - OldPage* prev = NULL; + Page* page = pages; + Page* prev = NULL; while (task_index < num_tasks) { if (page_index % pages_per_task == 0) { partitions[task_index].head = page; @@ -231,8 +229,8 @@ task_index++) { const intptr_t pages_per_task = num_pages / num_tasks; for (intptr_t j = 0; j < pages_per_task; j++) { - OldPage* page = heap_->old_space()->AllocatePage(OldPage::kData, - /* link */ false); + Page* page = heap_->old_space()->AllocatePage(Page::kData, + /* link */ false); if (page == nullptr) { oom = true; @@ -333,12 +331,12 @@ // Free empty pages. for (intptr_t task_index = 0; task_index < num_tasks; task_index++) { - OldPage* page = partitions[task_index].tail->next(); + Page* page = partitions[task_index].tail->next(); while (page != NULL) { - OldPage* next = page->next(); + Page* next = page->next(); heap_->old_space()->IncreaseCapacityInWordsLocked( -(page->memory_->size() >> kWordSizeLog2)); - page->Deallocate(); + page->Deallocate(/*can_use_cache*/ true); page = next; } } @@ -385,12 +383,12 @@ if (planning_task >= num_tasks_) break; TIMELINE_FUNCTION_GC_DURATION(thread, "Plan"); - OldPage* head = partitions_[planning_task].head; + Page* head = partitions_[planning_task].head; free_page_ = head; free_current_ = head->object_start(); free_end_ = head->object_end(); - for (OldPage* page = head; page != NULL; page = page->next()) { + for (Page* page = head; page != NULL; page = page->next()) { PlanPage(page); } } @@ -402,12 +400,12 @@ if (sliding_task >= num_tasks_) break; TIMELINE_FUNCTION_GC_DURATION(thread, "Slide"); - OldPage* head = partitions_[sliding_task].head; + Page* head = partitions_[sliding_task].head; free_page_ = head; free_current_ = head->object_start(); free_end_ = head->object_end(); - for (OldPage* page = head; page != NULL; page = page->next()) { + for (Page* page = head; page != NULL; page = page->next()) { SlidePage(page); } @@ -431,7 +429,7 @@ switch (forwarding_task) { case 0: { TIMELINE_FUNCTION_GC_DURATION(thread, "ForwardLargePages"); - for (OldPage* large_page = + for (Page* large_page = isolate_group_->heap()->old_space()->large_pages_; large_page != NULL; large_page = large_page->next()) { large_page->VisitObjectPointers(compactor_); @@ -479,7 +477,7 @@ } } -void CompactorTask::PlanPage(OldPage* page) { +void CompactorTask::PlanPage(Page* page) { uword current = page->object_start(); uword end = page->object_end(); @@ -491,7 +489,7 @@ } } -void CompactorTask::SlidePage(OldPage* page) { +void CompactorTask::SlidePage(Page* page) { uword current = page->object_start(); uword end = page->object_end(); @@ -552,7 +550,7 @@ // to a new page. But if we exactly hit the end of the previous page // then free_current could be at the start of the next page, so we // subtract 1. - ASSERT(OldPage::Of(free_current_ - 1) != OldPage::Of(new_addr)); + ASSERT(Page::Of(free_current_ - 1) != Page::Of(new_addr)); intptr_t free_remaining = free_end_ - free_current_; // Add any leftover at the end of a page to the free list. if (free_remaining > 0) { @@ -593,7 +591,7 @@ void CompactorTask::PlanMoveToContiguousSize(intptr_t size) { // Move the free cursor to ensure 'size' bytes of contiguous space. - ASSERT(size <= kOldPageSize); + ASSERT(size <= kPageSize); // Check if the current free page has enough space. intptr_t free_remaining = free_end_ - free_current_; @@ -611,7 +609,7 @@ void GCCompactor::SetupImagePageBoundaries() { MallocGrowableArray<ImagePageRange> ranges(4); - OldPage* image_page = + Page* image_page = Dart::vm_isolate_group()->heap()->old_space()->image_pages_; while (image_page != NULL) { ImagePageRange range = {image_page->object_start(), @@ -656,7 +654,7 @@ } } - OldPage* page = OldPage::Of(old_target); + Page* page = Page::Of(old_target); ForwardingPage* forwarding_page = page->forwarding_page(); if (forwarding_page == NULL) { return; // Not moved (VM isolate, large page, code page). @@ -692,7 +690,7 @@ } } - OldPage* page = OldPage::Of(old_target); + Page* page = Page::Of(old_target); ForwardingPage* forwarding_page = page->forwarding_page(); if (forwarding_page == NULL) { return; // Not moved (VM isolate, large page, code page).
diff --git a/runtime/vm/heap/compactor.h b/runtime/vm/heap/compactor.h index cff7736..7b65ccd 100644 --- a/runtime/vm/heap/compactor.h +++ b/runtime/vm/heap/compactor.h
@@ -17,7 +17,7 @@ // Forward declarations. class FreeList; class Heap; -class OldPage; +class Page; // Implements a sliding compactor. class GCCompactor : public ValueObject, @@ -30,7 +30,7 @@ heap_(heap) {} ~GCCompactor() { free(image_page_ranges_); } - void Compact(OldPage* pages, FreeList* freelist, Mutex* mutex); + void Compact(Page* pages, FreeList* freelist, Mutex* mutex); private: friend class CompactorTask;
diff --git a/runtime/vm/heap/freelist.h b/runtime/vm/heap/freelist.h index 8040f5f..7f2700e 100644 --- a/runtime/vm/heap/freelist.h +++ b/runtime/vm/heap/freelist.h
@@ -133,7 +133,7 @@ return result; } - // Ensures OldPage::VisitObjects can successful walk over a partially + // Ensures Page::VisitObjects can successful walk over a partially // allocated bump region. void MakeIterable() { if (top_ < end_) {
diff --git a/runtime/vm/heap/heap.cc b/runtime/vm/heap/heap.cc index 092fadb..ecbb141 100644 --- a/runtime/vm/heap/heap.cc +++ b/runtime/vm/heap/heap.cc
@@ -88,10 +88,10 @@ // It is possible a GC doesn't clear enough space. // In that case, we must fall through and allocate into old space. - return AllocateOld(thread, size, OldPage::kData); + return AllocateOld(thread, size, Page::kData); } -uword Heap::AllocateOld(Thread* thread, intptr_t size, OldPage::PageType type) { +uword Heap::AllocateOld(Thread* thread, intptr_t size, Page::PageType type) { ASSERT(thread->no_safepoint_scope_depth() == 0); if (!thread->force_growth()) { CollectForDebugging(thread); @@ -221,7 +221,7 @@ } bool Heap::CodeContains(uword addr) const { - return old_space_.Contains(addr, OldPage::kExecutable); + return old_space_.Contains(addr, Page::kExecutable); } bool Heap::DataContains(uword addr) const { @@ -346,14 +346,14 @@ InstructionsPtr Heap::FindObjectInCodeSpace(FindObjectVisitor* visitor) const { // Only executable pages can have RawInstructions objects. - ObjectPtr raw_obj = old_space_.FindObject(visitor, OldPage::kExecutable); + ObjectPtr raw_obj = old_space_.FindObject(visitor, Page::kExecutable); ASSERT((raw_obj == Object::null()) || (raw_obj->GetClassId() == kInstructionsCid)); return static_cast<InstructionsPtr>(raw_obj); } ObjectPtr Heap::FindOldObject(FindObjectVisitor* visitor) const { - return old_space_.FindObject(visitor, OldPage::kData); + return old_space_.FindObject(visitor, Page::kData); } ObjectPtr Heap::FindNewObject(FindObjectVisitor* visitor) { @@ -432,7 +432,7 @@ } if (OS::GetCurrentMonotonicMicros() < deadline) { - SemiSpace::ClearCache(); + Page::ClearCache(); } } @@ -709,19 +709,6 @@ isolate_group->set_heap(std::move(heap)); } -const char* Heap::RegionName(Space space) { - switch (space) { - case kNew: - return "dart-newspace"; - case kOld: - return "dart-oldspace"; - case kCode: - return "dart-codespace"; - default: - UNREACHABLE(); - } -} - void Heap::AddRegionsToObjectSet(ObjectSet* set) const { new_space_.AddRegionsToObjectSet(set); old_space_.AddRegionsToObjectSet(set);
diff --git a/runtime/vm/heap/heap.h b/runtime/vm/heap/heap.h index e1db075..a5f65eb 100644 --- a/runtime/vm/heap/heap.h +++ b/runtime/vm/heap/heap.h
@@ -65,13 +65,13 @@ case kNew: // Do not attempt to allocate very large objects in new space. if (!IsAllocatableInNewSpace(size)) { - return AllocateOld(thread, size, OldPage::kData); + return AllocateOld(thread, size, Page::kData); } return AllocateNew(thread, size); case kOld: - return AllocateOld(thread, size, OldPage::kData); + return AllocateOld(thread, size, Page::kData); case kCode: - return AllocateOld(thread, size, OldPage::kExecutable); + return AllocateOld(thread, size, Page::kExecutable); default: UNREACHABLE(); } @@ -150,9 +150,6 @@ intptr_t max_new_gen_words, intptr_t max_old_gen_words); - // Returns a suitable name for a VM region in the heap. - static const char* RegionName(Space space); - // Verify that all pointers in the heap point to the heap. bool Verify(MarkExpectation mark_expectation = kForbidMarked); @@ -324,7 +321,7 @@ intptr_t max_old_gen_words); uword AllocateNew(Thread* thread, intptr_t size); - uword AllocateOld(Thread* thread, intptr_t size, OldPage::PageType type); + uword AllocateOld(Thread* thread, intptr_t size, Page::PageType type); // Visit all pointers. Caller must ensure concurrent sweeper is not running, // and the visitor must not allocate.
diff --git a/runtime/vm/heap/heap_sources.gni b/runtime/vm/heap/heap_sources.gni index a8e8d99..39a2ec9 100644 --- a/runtime/vm/heap/heap_sources.gni +++ b/runtime/vm/heap/heap_sources.gni
@@ -17,6 +17,8 @@ "heap.h", "marker.cc", "marker.h", + "page.cc", + "page.h", "pages.cc", "pages.h", "pointer_block.cc",
diff --git a/runtime/vm/heap/marker.cc b/runtime/vm/heap/marker.cc index 6113430..4515324 100644 --- a/runtime/vm/heap/marker.cc +++ b/runtime/vm/heap/marker.cc
@@ -376,7 +376,7 @@ static bool TryAcquireMarkBit(ObjectPtr raw_obj) { if (FLAG_write_protect_code && raw_obj->IsInstructions()) { // A non-writable alias mapping may exist for instruction pages. - raw_obj = OldPage::ToWritable(raw_obj); + raw_obj = Page::ToWritable(raw_obj); } if (!sync) { raw_obj->untag()->SetMarkBitUnsynchronized(); @@ -495,7 +495,7 @@ root_slices_finished_ = 0; root_slices_count_ = kNumFixedRootSlices; new_page_ = heap_->new_space()->head(); - for (NewPage* p = new_page_; p != nullptr; p = p->next()) { + for (Page* p = new_page_; p != nullptr; p = p->next()) { root_slices_count_++; } @@ -518,7 +518,7 @@ break; } default: { - NewPage* page; + Page* page; { MonitorLocker ml(&root_slices_monitor_); page = new_page_;
diff --git a/runtime/vm/heap/marker.h b/runtime/vm/heap/marker.h index 6cc2653..2888017 100644 --- a/runtime/vm/heap/marker.h +++ b/runtime/vm/heap/marker.h
@@ -20,7 +20,7 @@ class PageSpace; template <bool sync> class MarkingVisitorBase; -class NewPage; +class Page; class Thread; // The class GCMarker is used to mark reachable old generation objects as part @@ -73,7 +73,7 @@ GCLinkedLists global_list_; MarkingVisitorBase<true>** visitors_; - NewPage* new_page_; + Page* new_page_; Monitor root_slices_monitor_; RelaxedAtomic<intptr_t> root_slices_started_; intptr_t root_slices_finished_;
diff --git a/runtime/vm/heap/page.cc b/runtime/vm/heap/page.cc new file mode 100644 index 0000000..a2eba82 --- /dev/null +++ b/runtime/vm/heap/page.cc
@@ -0,0 +1,286 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "vm/heap/page.h" + +#include "platform/assert.h" +#include "platform/leak_sanitizer.h" +#include "vm/dart.h" +#include "vm/heap/become.h" +#include "vm/heap/compactor.h" +#include "vm/heap/marker.h" +#include "vm/heap/safepoint.h" +#include "vm/heap/sweeper.h" +#include "vm/lockers.h" +#include "vm/log.h" +#include "vm/object.h" +#include "vm/object_set.h" +#include "vm/os_thread.h" +#include "vm/virtual_memory.h" + +namespace dart { + +// This cache needs to be at least as big as FLAG_new_gen_semi_max_size or +// munmap will noticably impact performance. +static constexpr intptr_t kPageCacheCapacity = 8 * kWordSize; +static Mutex* page_cache_mutex = nullptr; +static VirtualMemory* page_cache[kPageCacheCapacity] = {nullptr}; +static intptr_t page_cache_size = 0; + +void Page::Init() { + ASSERT(page_cache_mutex == nullptr); + page_cache_mutex = new Mutex(NOT_IN_PRODUCT("page_cache_mutex")); +} + +void Page::ClearCache() { + MutexLocker ml(page_cache_mutex); + ASSERT(page_cache_size >= 0); + ASSERT(page_cache_size <= kPageCacheCapacity); + while (page_cache_size > 0) { + delete page_cache[--page_cache_size]; + } +} + +void Page::Cleanup() { + ClearCache(); + delete page_cache_mutex; + page_cache_mutex = nullptr; +} + +intptr_t Page::CachedSize() { + MutexLocker ml(page_cache_mutex); + return page_cache_size * kPageSize; +} + +Page* Page::Allocate(intptr_t size, PageType type, bool can_use_cache) { + const bool executable = type == kExecutable; + const bool compressed = !executable; + const char* name = executable ? "dart-code" : "dart-heap"; + + VirtualMemory* memory = nullptr; + if (can_use_cache) { + // We don't automatically use the cache based on size and type because a + // large page that happens to be the same size as a regular page can't + // use the cache. Large pages are expected to be zeroed on allocation but + // cached pages are dirty. + ASSERT(size == kPageSize); + ASSERT(!executable); + MutexLocker ml(page_cache_mutex); + ASSERT(page_cache_size >= 0); + ASSERT(page_cache_size <= kPageCacheCapacity); + if (page_cache_size > 0) { + memory = page_cache[--page_cache_size]; + } + } + if (memory == nullptr) { + memory = VirtualMemory::AllocateAligned(size, kPageSize, executable, + compressed, name); + } + if (memory == nullptr) { + return nullptr; // Out of memory. + } + + if (type == kNew) { +#if defined(DEBUG) + memset(memory->address(), Heap::kZapByte, size); +#endif + // Initialized by generated code. + MSAN_UNPOISON(memory->address(), size); + } else { + // We don't zap old-gen because we rely on implicit zero-initialization + // of large typed data arrays. + } + + Page* result = reinterpret_cast<Page*>(memory->address()); + ASSERT(result != NULL); + result->type_ = type; + result->memory_ = memory; + result->next_ = nullptr; + result->forwarding_page_ = nullptr; + result->card_table_ = nullptr; + result->progress_bar_ = 0; + result->owner_ = nullptr; + result->top_ = 0; + result->end_ = 0; + result->survivor_end_ = 0; + result->resolved_top_ = 0; + + if (type == kNew) { + uword top = result->object_start(); + uword end = + memory->end() - kNewObjectAlignmentOffset - kAllocationRedZoneSize; + result->top_ = top; + result->end_ = end; + result->survivor_end_ = top; + result->resolved_top_ = top; + } + + LSAN_REGISTER_ROOT_REGION(result, sizeof(*result)); + + return result; +} + +void Page::Deallocate(bool can_use_cache) { + if (is_image_page()) { + delete memory_; + // For a heap page from a snapshot, the Page object lives in the malloc + // heap rather than the page itself. + free(this); + return; + } + + free(card_table_); + + LSAN_UNREGISTER_ROOT_REGION(this, sizeof(*this)); + + VirtualMemory* memory = memory_; + if (can_use_cache) { + ASSERT(memory->size() == kPageSize); + ASSERT(type_ != kExecutable); + MutexLocker ml(page_cache_mutex); + ASSERT(page_cache_size >= 0); + ASSERT(page_cache_size <= kPageCacheCapacity); + if (page_cache_size < kPageCacheCapacity) { + intptr_t size = memory->size(); +#if defined(DEBUG) + if (type_ == kNew) { + memset(memory->address(), Heap::kZapByte, size); + } else { + // We don't zap old-gen because we rely on implicit zero-initialization + // of large typed data arrays. + } +#endif + MSAN_POISON(memory->address(), size); + page_cache[page_cache_size++] = memory; + memory = nullptr; + } + } + delete memory; +} + +void Page::VisitObjects(ObjectVisitor* visitor) const { + ASSERT(Thread::Current()->IsAtSafepoint()); + NoSafepointScope no_safepoint; + uword obj_addr = object_start(); + uword end_addr = object_end(); + while (obj_addr < end_addr) { + ObjectPtr raw_obj = UntaggedObject::FromAddr(obj_addr); + visitor->VisitObject(raw_obj); + obj_addr += raw_obj->untag()->HeapSize(); + } + ASSERT(obj_addr == end_addr); +} + +void Page::VisitObjectPointers(ObjectPointerVisitor* visitor) const { + ASSERT(Thread::Current()->IsAtSafepoint() || + (Thread::Current()->task_kind() == Thread::kCompactorTask) || + (Thread::Current()->task_kind() == Thread::kMarkerTask)); + NoSafepointScope no_safepoint; + uword obj_addr = object_start(); + uword end_addr = object_end(); + while (obj_addr < end_addr) { + ObjectPtr raw_obj = UntaggedObject::FromAddr(obj_addr); + obj_addr += raw_obj->untag()->VisitPointers(visitor); + } + ASSERT(obj_addr == end_addr); +} + +void Page::VisitRememberedCards(ObjectPointerVisitor* visitor) { + ASSERT(Thread::Current()->IsAtSafepoint() || + (Thread::Current()->task_kind() == Thread::kScavengerTask)); + NoSafepointScope no_safepoint; + + if (card_table_ == NULL) { + return; + } + + ArrayPtr obj = + static_cast<ArrayPtr>(UntaggedObject::FromAddr(object_start())); + ASSERT(obj->IsArray()); + ASSERT(obj->untag()->IsCardRemembered()); + CompressedObjectPtr* obj_from = obj->untag()->from(); + CompressedObjectPtr* obj_to = + obj->untag()->to(Smi::Value(obj->untag()->length())); + uword heap_base = obj.heap_base(); + + const intptr_t size = card_table_size(); + for (;;) { + intptr_t i = progress_bar_.fetch_add(1); + if (i >= size) break; + + if (card_table_[i] != 0) { + CompressedObjectPtr* card_from = + reinterpret_cast<CompressedObjectPtr*>(this) + + (i << kSlotsPerCardLog2); + CompressedObjectPtr* card_to = + reinterpret_cast<CompressedObjectPtr*>(card_from) + + (1 << kSlotsPerCardLog2) - 1; + // Minus 1 because to is inclusive. + + if (card_from < obj_from) { + // First card overlaps with header. + card_from = obj_from; + } + if (card_to > obj_to) { + // Last card(s) may extend past the object. Array truncation can make + // this happen for more than one card. + card_to = obj_to; + } + + visitor->VisitCompressedPointers(heap_base, card_from, card_to); + + bool has_new_target = false; + for (CompressedObjectPtr* slot = card_from; slot <= card_to; slot++) { + if ((*slot)->IsNewObjectMayBeSmi()) { + has_new_target = true; + break; + } + } + + if (!has_new_target) { + card_table_[i] = 0; + } + } + } +} + +void Page::ResetProgressBar() { + progress_bar_ = 0; +} + +ObjectPtr Page::FindObject(FindObjectVisitor* visitor) const { + uword obj_addr = object_start(); + uword end_addr = object_end(); + if (visitor->VisitRange(obj_addr, end_addr)) { + while (obj_addr < end_addr) { + ObjectPtr raw_obj = UntaggedObject::FromAddr(obj_addr); + uword next_obj_addr = obj_addr + raw_obj->untag()->HeapSize(); + if (visitor->VisitRange(obj_addr, next_obj_addr) && + raw_obj->untag()->FindObject(visitor)) { + return raw_obj; // Found object, return it. + } + obj_addr = next_obj_addr; + } + ASSERT(obj_addr == end_addr); + } + return Object::null(); +} + +void Page::WriteProtect(bool read_only) { + ASSERT(!is_image_page()); + + VirtualMemory::Protection prot; + if (read_only) { + if ((type_ == kExecutable) && (memory_->AliasOffset() == 0)) { + prot = VirtualMemory::kReadExecute; + } else { + prot = VirtualMemory::kReadOnly; + } + } else { + prot = VirtualMemory::kReadWrite; + } + memory_->Protect(prot); +} + +} // namespace dart
diff --git a/runtime/vm/heap/page.h b/runtime/vm/heap/page.h new file mode 100644 index 0000000..5aba1ae --- /dev/null +++ b/runtime/vm/heap/page.h
@@ -0,0 +1,340 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#ifndef RUNTIME_VM_HEAP_PAGE_H_ +#define RUNTIME_VM_HEAP_PAGE_H_ + +#include "platform/atomic.h" +#include "vm/globals.h" +#include "vm/heap/spaces.h" +#include "vm/pointer_tagging.h" +#include "vm/raw_object.h" +#include "vm/virtual_memory.h" + +namespace dart { + +class ForwardingPage; +class ObjectVisitor; +class ObjectPointerVisitor; +class FindObjectVisitor; +class Thread; + +// Pages are allocated with kPageSize alignment so that the Page of any object +// can be computed by masking the object with kPageMask. This does not apply to +// image pages, whose address is choosen by the system loader rather than the +// Dart VM. +static constexpr intptr_t kPageSize = 512 * KB; +static constexpr intptr_t kPageSizeInWords = kPageSize / kWordSize; +static constexpr intptr_t kPageMask = ~(kPageSize - 1); + +// See ForwardingBlock and CountingBlock. +static constexpr intptr_t kBitVectorWordsPerBlock = 1; +static constexpr intptr_t kBlockSize = + kObjectAlignment * kBitsPerWord * kBitVectorWordsPerBlock; +static constexpr intptr_t kBlockMask = ~(kBlockSize - 1); +static constexpr intptr_t kBlocksPerPage = kPageSize / kBlockSize; + +// Simplify initialization in allocation stubs by ensuring it is safe +// to overshoot the object end by up to kAllocationRedZoneSize. (Just as the +// stack red zone allows one to overshoot the stack pointer.) +static constexpr intptr_t kAllocationRedZoneSize = kObjectAlignment; + +// A Page is the granuitary at which the Dart heap allocates memory from the OS. +// Pages are usually of size kPageSize, except large objects are allocated on +// their own Page sized to the object. +// +// +----------------------+ <- start +// | struct Page (header) | +// +----------------------+ +// | alignment gap | +// +----------------------+ <- object_start +// | objects | +// | ... | +// | ... | +// +----------------------+ <- object_end / top_ +// | available | +// +----------------------+ <- end_ +// | red zone or | +// | forwarding table | +// +----------------------+ <- memory_->end() +class Page { + public: + static void Init(); + static void ClearCache(); + static intptr_t CachedSize(); + static void Cleanup(); + + enum PageType : uword { kExecutable = 0, kData, kNew }; + + Page* next() const { return next_; } + void set_next(Page* next) { next_ = next; } + + uword start() const { return memory_->start(); } + uword end() const { return memory_->end(); } + bool Contains(uword addr) const { return memory_->Contains(addr); } + intptr_t AliasOffset() const { return memory_->AliasOffset(); } + + uword object_start() const { + return type_ == kNew ? new_object_start() : old_object_start(); + } + uword old_object_start() const { + return memory_->start() + OldObjectStartOffset(); + } + uword new_object_start() const { + return memory_->start() + NewObjectStartOffset(); + } + uword object_end() const { + if (owner_ != NULL) return owner_->top(); + return top_; + } + intptr_t used() const { return object_end() - object_start(); } + + ForwardingPage* forwarding_page() const { return forwarding_page_; } + void AllocateForwardingPage(); + + PageType type() const { return type_; } + + bool is_image_page() const { return !memory_->vm_owns_region(); } + + void VisitObjects(ObjectVisitor* visitor) const; + void VisitObjectPointers(ObjectPointerVisitor* visitor) const; + + ObjectPtr FindObject(FindObjectVisitor* visitor) const; + + void WriteProtect(bool read_only); + + static intptr_t OldObjectStartOffset() { + return Utils::RoundUp(sizeof(Page) - kOldObjectAlignmentOffset, + kObjectStartAlignment) + + kOldObjectAlignmentOffset; + } + static intptr_t NewObjectStartOffset() { + // Note weaker alignment because the bool/null offset tricks don't apply to + // new-space. + return Utils::RoundUp(sizeof(Page) - kNewObjectAlignmentOffset, + kObjectAlignment) + + kNewObjectAlignmentOffset; + } + + // Warning: This does not work for objects on image pages because image pages + // are not aligned. However, it works for objects on large pages, because + // only one object is allocated per large page. + static Page* Of(ObjectPtr obj) { + ASSERT(obj->IsHeapObject()); + return reinterpret_cast<Page*>(static_cast<uword>(obj) & kPageMask); + } + + // Warning: This does not work for addresses on image pages or on large pages. + static Page* Of(uword addr) { + return reinterpret_cast<Page*>(addr & kPageMask); + } + + // Warning: This does not work for objects on image pages. + static ObjectPtr ToExecutable(ObjectPtr obj) { + Page* page = Of(obj); + VirtualMemory* memory = page->memory_; + const intptr_t alias_offset = memory->AliasOffset(); + if (alias_offset == 0) { + return obj; // Not aliased. + } + uword addr = UntaggedObject::ToAddr(obj); + if (memory->Contains(addr)) { + return UntaggedObject::FromAddr(addr + alias_offset); + } + // obj is executable. + ASSERT(memory->ContainsAlias(addr)); + return obj; + } + + // Warning: This does not work for objects on image pages. + static ObjectPtr ToWritable(ObjectPtr obj) { + Page* page = Of(obj); + VirtualMemory* memory = page->memory_; + const intptr_t alias_offset = memory->AliasOffset(); + if (alias_offset == 0) { + return obj; // Not aliased. + } + uword addr = UntaggedObject::ToAddr(obj); + if (memory->ContainsAlias(addr)) { + return UntaggedObject::FromAddr(addr - alias_offset); + } + // obj is writable. + ASSERT(memory->Contains(addr)); + return obj; + } + + // 1 card = 128 slots. + static const intptr_t kSlotsPerCardLog2 = 7; + static const intptr_t kBytesPerCardLog2 = + kCompressedWordSizeLog2 + kSlotsPerCardLog2; + + intptr_t card_table_size() const { + return memory_->size() >> kBytesPerCardLog2; + } + + static intptr_t card_table_offset() { return OFFSET_OF(Page, card_table_); } + + void RememberCard(ObjectPtr const* slot) { + ASSERT(Contains(reinterpret_cast<uword>(slot))); + if (card_table_ == NULL) { + card_table_ = reinterpret_cast<uint8_t*>( + calloc(card_table_size(), sizeof(uint8_t))); + } + intptr_t offset = + reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this); + intptr_t index = offset >> kBytesPerCardLog2; + ASSERT((index >= 0) && (index < card_table_size())); + card_table_[index] = 1; + } + bool IsCardRemembered(ObjectPtr const* slot) { + ASSERT(Contains(reinterpret_cast<uword>(slot))); + if (card_table_ == NULL) { + return false; + } + intptr_t offset = + reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this); + intptr_t index = offset >> kBytesPerCardLog2; + ASSERT((index >= 0) && (index < card_table_size())); + return card_table_[index] != 0; + } +#if defined(DART_COMPRESSED_POINTERS) + void RememberCard(CompressedObjectPtr const* slot) { + ASSERT(Contains(reinterpret_cast<uword>(slot))); + if (card_table_ == NULL) { + card_table_ = reinterpret_cast<uint8_t*>( + calloc(card_table_size(), sizeof(uint8_t))); + } + intptr_t offset = + reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this); + intptr_t index = offset >> kBytesPerCardLog2; + ASSERT((index >= 0) && (index < card_table_size())); + card_table_[index] = 1; + } + bool IsCardRemembered(CompressedObjectPtr const* slot) { + ASSERT(Contains(reinterpret_cast<uword>(slot))); + if (card_table_ == NULL) { + return false; + } + intptr_t offset = + reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this); + intptr_t index = offset >> kBytesPerCardLog2; + ASSERT((index >= 0) && (index < card_table_size())); + return card_table_[index] != 0; + } +#endif + void VisitRememberedCards(ObjectPointerVisitor* visitor); + void ResetProgressBar(); + + Thread* owner() const { + return owner_; + } + + // Remember the limit to which objects have been copied. + void RecordSurvivors() { + survivor_end_ = object_end(); + } + + // Move survivor end to the end of the to_ space, making all surviving + // objects candidates for promotion next time. + void EarlyTenure() { + survivor_end_ = end_; + } + + uword promo_candidate_words() const { + return (survivor_end_ - object_start()) / kWordSize; + } + + void Acquire(Thread* thread) { + ASSERT(owner_ == nullptr); + owner_ = thread; + thread->set_top(top_); + thread->set_end(end_); + } + void Release(Thread* thread) { + ASSERT(owner_ == thread); + owner_ = nullptr; + top_ = thread->top(); + thread->set_top(0); + thread->set_end(0); + } + void Release() { + if (owner_ != nullptr) { + Release(owner_); + } + } + + uword TryAllocateGC(intptr_t size) { + ASSERT(owner_ == nullptr); + uword result = top_; + uword new_top = result + size; + if (LIKELY(new_top <= end_)) { + top_ = new_top; + return result; + } + return 0; + } + + void Unallocate(uword addr, intptr_t size) { + ASSERT((addr + size) == top_); + top_ -= size; + } + + bool IsSurvivor(uword raw_addr) const { + return raw_addr < survivor_end_; + } + bool IsResolved() const { + return top_ == resolved_top_; + } + + private: + void set_object_end(uword value) { + ASSERT((value & kObjectAlignmentMask) == kOldObjectAlignmentOffset); + top_ = value; + } + + // Returns NULL on OOM. + static Page* Allocate(intptr_t size, PageType type, bool can_use_cache); + + // Deallocate the virtual memory backing this page. The page pointer to this + // page becomes immediately inaccessible. + void Deallocate(bool can_use_cache); + + PageType type_; + VirtualMemory* memory_; + Page* next_; + ForwardingPage* forwarding_page_; + uint8_t* card_table_; // Remembered set, not marking. + RelaxedAtomic<intptr_t> progress_bar_; + + // The thread using this page for allocation, otherwise NULL. + Thread* owner_; + + // The address of the next allocation. If owner is non-NULL, this value is + // stale and the current value is at owner->top_. Called "NEXT" in the + // original Cheney paper. + uword top_; + + // The address after the last allocatable byte in this page. + uword end_; + + // Objects below this address have survived a scavenge. + uword survivor_end_; + + // A pointer to the first unprocessed object. Resolution completes when this + // value meets the allocation top. Called "SCAN" in the original Cheney paper. + uword resolved_top_; + + template <bool> + friend class ScavengerVisitorBase; + friend class SemiSpace; + friend class PageSpace; + friend class GCCompactor; + + DISALLOW_ALLOCATION(); + DISALLOW_IMPLICIT_CONSTRUCTORS(Page); +}; + +} // namespace dart + +#endif // RUNTIME_VM_HEAP_PAGE_H_
diff --git a/runtime/vm/heap/pages.cc b/runtime/vm/heap/pages.cc index 25e5ad3..a928109 100644 --- a/runtime/vm/heap/pages.cc +++ b/runtime/vm/heap/pages.cc
@@ -43,180 +43,6 @@ "Print free list statistics after a GC"); DEFINE_FLAG(bool, log_growth, false, "Log PageSpace growth policy decisions."); -OldPage* OldPage::Allocate(intptr_t size_in_words, - PageType type, - const char* name) { - const bool executable = type == kExecutable; - const bool compressed = !executable; - - VirtualMemory* memory = VirtualMemory::AllocateAligned( - size_in_words << kWordSizeLog2, kOldPageSize, executable, compressed, - name); - if (memory == NULL) { - return NULL; - } - - OldPage* result = reinterpret_cast<OldPage*>(memory->address()); - ASSERT(result != NULL); - result->memory_ = memory; - result->next_ = NULL; - result->used_in_bytes_ = 0; - result->forwarding_page_ = NULL; - result->card_table_ = NULL; - result->progress_bar_ = 0; - result->type_ = type; - - LSAN_REGISTER_ROOT_REGION(result, sizeof(*result)); - - return result; -} - -void OldPage::Deallocate() { - if (card_table_ != NULL) { - free(card_table_); - card_table_ = NULL; - } - - bool image_page = is_image_page(); - - if (!image_page) { - LSAN_UNREGISTER_ROOT_REGION(this, sizeof(*this)); - } - - // For a regular heap pages, the memory for this object will become - // unavailable after the delete below. - delete memory_; - - // For a heap page from a snapshot, the OldPage object lives in the malloc - // heap rather than the page itself. - if (image_page) { - free(this); - } -} - -void OldPage::VisitObjects(ObjectVisitor* visitor) const { - ASSERT(Thread::Current()->IsAtSafepoint()); - NoSafepointScope no_safepoint; - uword obj_addr = object_start(); - uword end_addr = object_end(); - while (obj_addr < end_addr) { - ObjectPtr raw_obj = UntaggedObject::FromAddr(obj_addr); - visitor->VisitObject(raw_obj); - obj_addr += raw_obj->untag()->HeapSize(); - } - ASSERT(obj_addr == end_addr); -} - -void OldPage::VisitObjectPointers(ObjectPointerVisitor* visitor) const { - ASSERT(Thread::Current()->IsAtSafepoint() || - (Thread::Current()->task_kind() == Thread::kCompactorTask)); - NoSafepointScope no_safepoint; - uword obj_addr = object_start(); - uword end_addr = object_end(); - while (obj_addr < end_addr) { - ObjectPtr raw_obj = UntaggedObject::FromAddr(obj_addr); - obj_addr += raw_obj->untag()->VisitPointers(visitor); - } - ASSERT(obj_addr == end_addr); -} - -void OldPage::VisitRememberedCards(ObjectPointerVisitor* visitor) { - ASSERT(Thread::Current()->IsAtSafepoint() || - (Thread::Current()->task_kind() == Thread::kScavengerTask)); - NoSafepointScope no_safepoint; - - if (card_table_ == NULL) { - return; - } - - ArrayPtr obj = - static_cast<ArrayPtr>(UntaggedObject::FromAddr(object_start())); - ASSERT(obj->IsArray()); - ASSERT(obj->untag()->IsCardRemembered()); - CompressedObjectPtr* obj_from = obj->untag()->from(); - CompressedObjectPtr* obj_to = - obj->untag()->to(Smi::Value(obj->untag()->length())); - uword heap_base = obj.heap_base(); - - const intptr_t size = card_table_size(); - for (;;) { - intptr_t i = progress_bar_.fetch_add(1); - if (i >= size) break; - - if (card_table_[i] != 0) { - CompressedObjectPtr* card_from = - reinterpret_cast<CompressedObjectPtr*>(this) + - (i << kSlotsPerCardLog2); - CompressedObjectPtr* card_to = - reinterpret_cast<CompressedObjectPtr*>(card_from) + - (1 << kSlotsPerCardLog2) - 1; - // Minus 1 because to is inclusive. - - if (card_from < obj_from) { - // First card overlaps with header. - card_from = obj_from; - } - if (card_to > obj_to) { - // Last card(s) may extend past the object. Array truncation can make - // this happen for more than one card. - card_to = obj_to; - } - - visitor->VisitCompressedPointers(heap_base, card_from, card_to); - - bool has_new_target = false; - for (CompressedObjectPtr* slot = card_from; slot <= card_to; slot++) { - if ((*slot)->IsNewObjectMayBeSmi()) { - has_new_target = true; - break; - } - } - - if (!has_new_target) { - card_table_[i] = 0; - } - } - } -} - -void OldPage::ResetProgressBar() { - progress_bar_ = 0; -} - -ObjectPtr OldPage::FindObject(FindObjectVisitor* visitor) const { - uword obj_addr = object_start(); - uword end_addr = object_end(); - if (visitor->VisitRange(obj_addr, end_addr)) { - while (obj_addr < end_addr) { - ObjectPtr raw_obj = UntaggedObject::FromAddr(obj_addr); - uword next_obj_addr = obj_addr + raw_obj->untag()->HeapSize(); - if (visitor->VisitRange(obj_addr, next_obj_addr) && - raw_obj->untag()->FindObject(visitor)) { - return raw_obj; // Found object, return it. - } - obj_addr = next_obj_addr; - } - ASSERT(obj_addr == end_addr); - } - return Object::null(); -} - -void OldPage::WriteProtect(bool read_only) { - ASSERT(!is_image_page()); - - VirtualMemory::Protection prot; - if (read_only) { - if ((type_ == kExecutable) && (memory_->AliasOffset() == 0)) { - prot = VirtualMemory::kReadExecute; - } else { - prot = VirtualMemory::kReadOnly; - } - } else { - prot = VirtualMemory::kReadWrite; - } - memory_->Protect(prot); -} - // The initial estimate of how many words we can mark per microsecond (usage // before / mark-sweep time). This is a conservative value observed running // Flutter on a Nexus 4. After the first mark-sweep, we instead use a value @@ -266,21 +92,21 @@ ml.Wait(); } } - FreePages(pages_); - FreePages(exec_pages_); - FreePages(large_pages_); - FreePages(image_pages_); + FreePages(pages_, /*can_use_cache*/ true); + FreePages(exec_pages_, /*can_use_cache*/ false); + FreePages(large_pages_, /*can_use_cache*/ false); + FreePages(image_pages_, /*can_use_cache*/ false); ASSERT(marker_ == NULL); delete[] freelists_; } intptr_t PageSpace::LargePageSizeInWordsFor(intptr_t size) { - intptr_t page_size = Utils::RoundUp(size + OldPage::ObjectStartOffset(), + intptr_t page_size = Utils::RoundUp(size + Page::OldObjectStartOffset(), VirtualMemory::PageSize()); return page_size >> kWordSizeLog2; } -void PageSpace::AddPageLocked(OldPage* page) { +void PageSpace::AddPageLocked(Page* page) { if (pages_ == nullptr) { pages_ = page; } else { @@ -289,7 +115,7 @@ pages_tail_ = page; } -void PageSpace::AddLargePageLocked(OldPage* page) { +void PageSpace::AddLargePageLocked(Page* page) { if (large_pages_ == nullptr) { large_pages_ = page; } else { @@ -298,7 +124,7 @@ large_pages_tail_ = page; } -void PageSpace::AddExecPageLocked(OldPage* page) { +void PageSpace::AddExecPageLocked(Page* page) { if (exec_pages_ == nullptr) { exec_pages_ = page; } else { @@ -313,7 +139,7 @@ exec_pages_tail_ = page; } -void PageSpace::RemovePageLocked(OldPage* page, OldPage* previous_page) { +void PageSpace::RemovePageLocked(Page* page, Page* previous_page) { if (previous_page != NULL) { previous_page->set_next(page->next()); } else { @@ -324,7 +150,7 @@ } } -void PageSpace::RemoveLargePageLocked(OldPage* page, OldPage* previous_page) { +void PageSpace::RemoveLargePageLocked(Page* page, Page* previous_page) { if (previous_page != NULL) { previous_page->set_next(page->next()); } else { @@ -335,7 +161,7 @@ } } -void PageSpace::RemoveExecPageLocked(OldPage* page, OldPage* previous_page) { +void PageSpace::RemoveExecPageLocked(Page* page, Page* previous_page) { if (previous_page != NULL) { previous_page->set_next(page->next()); } else { @@ -346,20 +172,19 @@ } } -OldPage* PageSpace::AllocatePage(OldPage::PageType type, bool link) { +Page* PageSpace::AllocatePage(Page::PageType type, bool link) { { MutexLocker ml(&pages_lock_); - if (!CanIncreaseCapacityInWordsLocked(kOldPageSizeInWords)) { + if (!CanIncreaseCapacityInWordsLocked(kPageSizeInWords)) { return nullptr; } - IncreaseCapacityInWordsLocked(kOldPageSizeInWords); + IncreaseCapacityInWordsLocked(kPageSizeInWords); } - const bool is_exec = (type == OldPage::kExecutable); - const char* name = Heap::RegionName(is_exec ? Heap::kCode : Heap::kOld); - OldPage* page = OldPage::Allocate(kOldPageSizeInWords, type, name); + const bool is_exec = (type == Page::kExecutable); + Page* page = Page::Allocate(kPageSize, type, /*can_use_cache*/ !is_exec); if (page == nullptr) { RELEASE_ASSERT(!FLAG_abort_on_oom); - IncreaseCapacityInWords(-kOldPageSizeInWords); + IncreaseCapacityInWords(-kPageSizeInWords); return nullptr; } @@ -373,14 +198,14 @@ } page->set_object_end(page->memory_->end()); - if ((type != OldPage::kExecutable) && (heap_ != nullptr) && + if ((type != Page::kExecutable) && (heap_ != nullptr) && (!heap_->is_vm_isolate())) { page->AllocateForwardingPage(); } return page; } -OldPage* PageSpace::AllocateLargePage(intptr_t size, OldPage::PageType type) { +Page* PageSpace::AllocateLargePage(intptr_t size, Page::PageType type) { const intptr_t page_size_in_words = LargePageSizeInWordsFor(size); { MutexLocker ml(&pages_lock_); @@ -389,9 +214,9 @@ } IncreaseCapacityInWordsLocked(page_size_in_words); } - const bool is_exec = (type == OldPage::kExecutable); - const char* name = Heap::RegionName(is_exec ? Heap::kCode : Heap::kOld); - OldPage* page = OldPage::Allocate(page_size_in_words, type, name); + const bool is_exec = (type == Page::kExecutable); + Page* page = Page::Allocate(page_size_in_words << kWordSizeLog2, type, + /*can_use_cache*/ false); MutexLocker ml(&pages_lock_); if (page == nullptr) { @@ -415,7 +240,7 @@ return page; } -void PageSpace::TruncateLargePage(OldPage* page, +void PageSpace::TruncateLargePage(Page* page, intptr_t new_object_size_in_bytes) { const intptr_t old_object_size_in_bytes = page->object_end() - page->object_start(); @@ -431,8 +256,8 @@ } } -void PageSpace::FreePage(OldPage* page, OldPage* previous_page) { - bool is_exec = (page->type() == OldPage::kExecutable); +void PageSpace::FreePage(Page* page, Page* previous_page) { + bool is_exec = (page->type() == Page::kExecutable); { MutexLocker ml(&pages_lock_); IncreaseCapacityInWordsLocked(-(page->memory_->size() >> kWordSizeLog2)); @@ -442,30 +267,29 @@ RemovePageLocked(page, previous_page); } } - // TODO(iposva): Consider adding to a pool of empty pages. - page->Deallocate(); + page->Deallocate(/*can_use_cache*/ !is_exec); } -void PageSpace::FreeLargePage(OldPage* page, OldPage* previous_page) { - ASSERT(page->type() != OldPage::kExecutable); +void PageSpace::FreeLargePage(Page* page, Page* previous_page) { + ASSERT(page->type() != Page::kExecutable); MutexLocker ml(&pages_lock_); IncreaseCapacityInWordsLocked(-(page->memory_->size() >> kWordSizeLog2)); RemoveLargePageLocked(page, previous_page); - page->Deallocate(); + page->Deallocate(/*can_use_cache*/ false); } -void PageSpace::FreePages(OldPage* pages) { - OldPage* page = pages; +void PageSpace::FreePages(Page* pages, bool can_use_cache) { + Page* page = pages; while (page != NULL) { - OldPage* next = page->next(); - page->Deallocate(); + Page* next = page->next(); + page->Deallocate(can_use_cache); page = next; } } uword PageSpace::TryAllocateInFreshPage(intptr_t size, FreeList* freelist, - OldPage::PageType type, + Page::PageType type, GrowthPolicy growth_policy, bool is_locked) { ASSERT(Heap::IsAllocatableViaFreeLists(size)); @@ -474,7 +298,7 @@ ASSERT(!Thread::Current()->force_growth()); if (heap_ != nullptr) { // Some unit tests. heap_->CheckConcurrentMarking(Thread::Current(), GCReason::kOldSpace, - kOldPageSize); + kPageSize); } } @@ -482,10 +306,10 @@ SpaceUsage after_allocation = GetCurrentUsage(); after_allocation.used_in_words += size >> kWordSizeLog2; // Can we grow by one page? - after_allocation.capacity_in_words += kOldPageSizeInWords; + after_allocation.capacity_in_words += kPageSizeInWords; if (growth_policy == kForceGrowth || !page_space_controller_.ReachedHardThreshold(after_allocation)) { - OldPage* page = AllocatePage(type); + Page* page = AllocatePage(type); if (page == NULL) { return 0; } @@ -508,7 +332,7 @@ } uword PageSpace::TryAllocateInFreshLargePage(intptr_t size, - OldPage::PageType type, + Page::PageType type, GrowthPolicy growth_policy) { ASSERT(!Heap::IsAllocatableViaFreeLists(size)); @@ -532,7 +356,7 @@ after_allocation.capacity_in_words += page_size_in_words; if (growth_policy == kForceGrowth || !page_space_controller_.ReachedHardThreshold(after_allocation)) { - OldPage* page = AllocateLargePage(size, type); + Page* page = AllocateLargePage(size, type); if (page != NULL) { result = page->object_start(); // Note: usage_.capacity_in_words is increased by AllocateLargePage. @@ -544,7 +368,7 @@ uword PageSpace::TryAllocateInternal(intptr_t size, FreeList* freelist, - OldPage::PageType type, + Page::PageType type, GrowthPolicy growth_policy, bool is_protected, bool is_locked) { @@ -586,7 +410,7 @@ public: explicit BasePageIterator(const PageSpace* space) : space_(space) {} - OldPage* page() const { return page_; } + Page* page() const { return page_; } bool Done() const { return page_ == NULL; } @@ -630,7 +454,7 @@ const PageSpace* space_ = nullptr; List list_; - OldPage* page_ = nullptr; + Page* page_ = nullptr; }; // Provides unsafe access to all pages. Assumes pages are walkable. @@ -665,7 +489,7 @@ space_->MakeIterable(); page_ = space_->exec_pages_; } - OldPage* page() const { return page_; } + Page* page() const { return page_; } bool Done() const { return page_ == NULL; } void Advance() { ASSERT(!Done()); @@ -676,7 +500,7 @@ const PageSpace* space_; MutexLocker ml_; NoSafepointScope no_safepoint; - OldPage* page_; + Page* page_; }; void PageSpace::MakeIterable() const { @@ -740,8 +564,8 @@ return false; } -bool PageSpace::Contains(uword addr, OldPage::PageType type) const { - if (type == OldPage::kExecutable) { +bool PageSpace::Contains(uword addr, Page::PageType type) const { + if (type == Page::kExecutable) { // Fast path executable pages. for (ExclusiveCodePageIterator it(this); !it.Done(); it.Advance()) { if (it.page()->Contains(addr)) { @@ -760,8 +584,7 @@ bool PageSpace::DataContains(uword addr) const { for (ExclusivePageIterator it(this); !it.Done(); it.Advance()) { - if ((it.page()->type() != OldPage::kExecutable) && - it.page()->Contains(addr)) { + if ((it.page()->type() != Page::kExecutable) && it.page()->Contains(addr)) { return true; } } @@ -818,9 +641,9 @@ // Large pages may be added concurrently due to promotion in another scavenge // worker, so terminate the traversal when we hit the tail we saw while // holding the pages lock, instead of at NULL, otherwise we are racing when we - // read OldPage::next_ and OldPage::remembered_cards_. - OldPage* page; - OldPage* tail; + // read Page::next_ and Page::remembered_cards_. + Page* page; + Page* tail; { MutexLocker ml(&pages_lock_); page = large_pages_; @@ -834,14 +657,14 @@ } void PageSpace::ResetProgressBars() const { - for (OldPage* page = large_pages_; page != NULL; page = page->next()) { + for (Page* page = large_pages_; page != NULL; page = page->next()) { page->ResetProgressBar(); } } ObjectPtr PageSpace::FindObject(FindObjectVisitor* visitor, - OldPage::PageType type) const { - if (type == OldPage::kExecutable) { + Page::PageType type) const { + if (type == Page::kExecutable) { // Fast path executable pages. for (ExclusiveCodePageIterator it(this); !it.Done(); it.Advance()) { ObjectPtr obj = it.page()->FindObject(visitor); @@ -920,7 +743,7 @@ heap_map.AddProperty("freeClassId", static_cast<intptr_t>(kFreeListElement)); heap_map.AddProperty("unitSizeBytes", static_cast<intptr_t>(kObjectAlignment)); - heap_map.AddProperty("pageSizeBytes", kOldPageSizeInWords * kWordSize); + heap_map.AddProperty("pageSizeBytes", kPageSizeInWords * kWordSize); { JSONObject class_list(&heap_map, "classList"); isolate_group->class_table()->PrintToJSONObject(&class_list); @@ -933,7 +756,7 @@ MutexLocker ml(&pages_lock_); MakeIterable(); JSONArray all_pages(&heap_map, "pages"); - for (OldPage* page = pages_; page != NULL; page = page->next()) { + for (Page* page = pages_; page != NULL; page = page->next()) { JSONObject page_container(&all_pages); page_container.AddPropertyF("objectStart", "0x%" Px "", page->object_start()); @@ -941,7 +764,7 @@ HeapMapAsJSONVisitor printer(&page_map); page->VisitObjects(&printer); } - for (OldPage* page = exec_pages_; page != NULL; page = page->next()) { + for (Page* page = exec_pages_; page != NULL; page = page->next()) { JSONObject page_container(&all_pages); page_container.AddPropertyF("objectStart", "0x%" Px "", page->object_start()); @@ -958,15 +781,15 @@ MutexLocker ml(&pages_lock_); NoSafepointScope no_safepoint; // No need to go through all of the data pages first. - OldPage* page = exec_pages_; + Page* page = exec_pages_; while (page != NULL) { - ASSERT(page->type() == OldPage::kExecutable); + ASSERT(page->type() == Page::kExecutable); page->WriteProtect(read_only); page = page->next(); } page = large_pages_; while (page != NULL) { - if (page->type() == OldPage::kExecutable) { + if (page->type() == Page::kExecutable) { page->WriteProtect(read_only); } page = page->next(); @@ -1009,7 +832,7 @@ // Discount two pages to account for the newest data and code pages, whose // partial use doesn't indicate fragmentation. const intptr_t excess_in_words = - usage_.capacity_in_words - usage_.used_in_words - 2 * kOldPageSizeInWords; + usage_.capacity_in_words - usage_.used_in_words - 2 * kPageSizeInWords; const double excess_ratio = static_cast<double>(excess_in_words) / static_cast<double>(usage_.capacity_in_words); const bool fragmented = excess_ratio > 0.05; @@ -1073,7 +896,7 @@ uword addr = reinterpret_cast<uword>(oom_reservation_); intptr_t size = oom_reservation_->HeapSize(); oom_reservation_ = nullptr; - freelists_[OldPage::kData].Free(addr, size); + freelists_[Page::kData].Free(addr, size); } bool PageSpace::MarkReservation() { @@ -1089,7 +912,7 @@ void PageSpace::TryReserveForOOM() { if (oom_reservation_ == nullptr) { - uword addr = TryAllocate(kOOMReservationSize, OldPage::kData, + uword addr = TryAllocate(kOOMReservationSize, Page::kData, kForceGrowth /* Don't re-enter GC */); if (addr != 0) { oom_reservation_ = FreeListElement::AsElement(addr, kOOMReservationSize); @@ -1229,12 +1052,12 @@ // code protection. TIMELINE_FUNCTION_GC_DURATION(thread, "SweepExecutable"); GCSweeper sweeper; - OldPage* prev_page = NULL; - OldPage* page = exec_pages_; - FreeList* freelist = &freelists_[OldPage::kExecutable]; + Page* prev_page = NULL; + Page* page = exec_pages_; + FreeList* freelist = &freelists_[Page::kExecutable]; MutexLocker ml(freelist->mutex()); while (page != NULL) { - OldPage* next_page = page->next(); + Page* next_page = page->next(); bool page_in_use = sweeper.SweepPage(page, freelist, true /*is_locked*/); if (page_in_use) { prev_page = page; @@ -1313,17 +1136,17 @@ GCSweeper sweeper; MutexLocker ml(&pages_lock_); while (sweep_large_ != nullptr) { - OldPage* page = sweep_large_; + Page* page = sweep_large_; sweep_large_ = page->next(); page->set_next(nullptr); - ASSERT(page->type() == OldPage::kData); + ASSERT(page->type() == Page::kData); ml.Unlock(); intptr_t words_to_end = sweeper.SweepLargePage(page); intptr_t size; if (words_to_end == 0) { size = page->memory_->size(); - page->Deallocate(); + page->Deallocate(/*can_use_cache*/ false); ml.Lock(); IncreaseCapacityInWordsLocked(-(size >> kWordSizeLog2)); } else { @@ -1349,10 +1172,10 @@ MutexLocker ml(&pages_lock_); while (sweep_regular_ != nullptr) { - OldPage* page = sweep_regular_; + Page* page = sweep_regular_; sweep_regular_ = page->next(); page->set_next(nullptr); - ASSERT(page->type() == OldPage::kData); + ASSERT(page->type() == Page::kData); ml.Unlock(); // Cycle through the shards round-robin so that free space is roughly @@ -1363,7 +1186,7 @@ intptr_t size; if (!page_in_use) { size = page->memory_->size(); - page->Deallocate(); + page->Deallocate(/*can_use_cache*/ true); } ml.Lock(); @@ -1389,7 +1212,7 @@ void PageSpace::Compact(Thread* thread) { thread->isolate_group()->set_compaction_in_progress(true); GCCompactor compactor(thread, heap_); - compactor.Compact(pages_, &freelists_[OldPage::kData], &pages_lock_); + compactor.Compact(pages_, &freelists_[Page::kData], &pages_lock_); thread->isolate_group()->set_compaction_in_progress(false); if (FLAG_verify_after_gc) { @@ -1415,8 +1238,8 @@ // side-effect of populating the freelist with a large block. The next // bump allocation request will have a chance to consume that block. // TODO(koda): Could take freelist lock just once instead of twice. - return TryAllocateInFreshPage(size, freelist, OldPage::kData, - kForceGrowth, true /* is_locked*/); + return TryAllocateInFreshPage(size, freelist, Page::kData, kForceGrowth, + true /* is_locked*/); } intptr_t block_size = block->HeapSize(); if (remaining > 0) { @@ -1462,30 +1285,29 @@ } void PageSpace::SetupImagePage(void* pointer, uword size, bool is_executable) { - // Setup a OldPage so precompiled Instructions can be traversed. - // Instructions are contiguous at [pointer, pointer + size). OldPage + // Setup a Page so precompiled Instructions can be traversed. + // Instructions are contiguous at [pointer, pointer + size). Page // expects to find objects at [memory->start() + ObjectStartOffset, // memory->end()). - uword offset = OldPage::ObjectStartOffset(); + uword offset = Page::OldObjectStartOffset(); pointer = reinterpret_cast<void*>(reinterpret_cast<uword>(pointer) - offset); ASSERT(Utils::IsAligned(pointer, kObjectAlignment)); size += offset; VirtualMemory* memory = VirtualMemory::ForImagePage(pointer, size); ASSERT(memory != NULL); - OldPage* page = reinterpret_cast<OldPage*>(malloc(sizeof(OldPage))); + Page* page = reinterpret_cast<Page*>(malloc(sizeof(Page))); + page->type_ = is_executable ? Page::kExecutable : Page::kData; page->memory_ = memory; - page->next_ = NULL; - page->object_end_ = memory->end(); - page->used_in_bytes_ = page->object_end_ - page->object_start(); - page->forwarding_page_ = NULL; - page->card_table_ = NULL; + page->next_ = nullptr; + page->forwarding_page_ = nullptr; + page->card_table_ = nullptr; page->progress_bar_ = 0; - if (is_executable) { - page->type_ = OldPage::kExecutable; - } else { - page->type_ = OldPage::kData; - } + page->owner_ = nullptr; + page->top_ = memory->end(); + page->end_ = memory->end(); + page->survivor_end_ = 0; + page->resolved_top_ = 0; MutexLocker ml(&pages_lock_); page->next_ = image_pages_; @@ -1494,7 +1316,7 @@ bool PageSpace::IsObjectFromImagePages(dart::ObjectPtr object) { uword object_addr = UntaggedObject::ToAddr(object); - OldPage* image_page = image_pages_; + Page* image_page = image_pages_; while (image_page != nullptr) { if (image_page->Contains(object_addr)) { return true; @@ -1585,7 +1407,7 @@ (static_cast<intptr_t>(after.CombinedUsedInWords() / desired_utilization_) - (after.CombinedUsedInWords())) / - kOldPageSizeInWords; + kPageSizeInWords; if (garbage_ratio == 0) { // No garbage in the previous cycle so it would be hard to compute a // grow_heap size based on estimated garbage so we use growth ratio @@ -1600,8 +1422,8 @@ intptr_t local_grow_heap = 0; while (min < max) { local_grow_heap = (max + min) / 2; - const intptr_t limit = after.CombinedUsedInWords() + - (local_grow_heap * kOldPageSizeInWords); + const intptr_t limit = + after.CombinedUsedInWords() + (local_grow_heap * kPageSizeInWords); const intptr_t allocated_before_next_gc = limit - (after.CombinedUsedInWords()); const double estimated_garbage = k * allocated_before_next_gc; @@ -1630,7 +1452,7 @@ ASSERT(grow_heap >= 0); // Fraction of asymptote used. double f = static_cast<double>(after.CombinedUsedInWords() + - (kOldPageSizeInWords * grow_heap)) / + (kPageSizeInWords * grow_heap)) / static_cast<double>(max_capacity_in_words); ASSERT(f >= 0.0); // Increase weight at the high end. @@ -1641,7 +1463,7 @@ // Discount growth more the closer we get to the desired asymptote. grow_heap = static_cast<intptr_t>(grow_heap * f); // Minimum growth step after reaching the asymptote. - intptr_t min_step = (2 * MB) / kOldPageSize; + intptr_t min_step = (2 * MB) / kPageSize; grow_heap = Utils::Maximum(min_step, grow_heap); } @@ -1658,7 +1480,7 @@ growth_in_pages = (static_cast<intptr_t>(after.CombinedUsedInWords() / desired_utilization_) - (after.CombinedUsedInWords())) / - kOldPageSizeInWords; + kPageSizeInWords; } // Apply growth cap. @@ -1674,7 +1496,7 @@ const char* reason) { // Save final threshold compared before growing. intptr_t threshold = - after.CombinedUsedInWords() + (kOldPageSizeInWords * growth_in_pages); + after.CombinedUsedInWords() + (kPageSizeInWords * growth_in_pages); #if defined(TARGET_ARCH_IA32) bool concurrent_mark = false; @@ -1691,7 +1513,7 @@ // Set a tight idle threshold. idle_gc_threshold_in_words_ = - after.CombinedUsedInWords() + (2 * kOldPageSizeInWords); + after.CombinedUsedInWords() + (2 * kPageSizeInWords); #if defined(SUPPORT_TIMELINE) Thread* thread = Thread::Current();
diff --git a/runtime/vm/heap/pages.h b/runtime/vm/heap/pages.h index 7c76f3e..406cded 100644 --- a/runtime/vm/heap/pages.h +++ b/runtime/vm/heap/pages.h
@@ -8,6 +8,7 @@ #include "platform/atomic.h" #include "vm/globals.h" #include "vm/heap/freelist.h" +#include "vm/heap/page.h" #include "vm/heap/spaces.h" #include "vm/lockers.h" #include "vm/ring_buffer.h" @@ -26,196 +27,6 @@ class ForwardingPage; class GCMarker; -static constexpr intptr_t kOldPageSize = 512 * KB; -static constexpr intptr_t kOldPageSizeInWords = kOldPageSize / kWordSize; -static constexpr intptr_t kOldPageMask = ~(kOldPageSize - 1); - -static constexpr intptr_t kBitVectorWordsPerBlock = 1; -static constexpr intptr_t kBlockSize = - kObjectAlignment * kBitsPerWord * kBitVectorWordsPerBlock; -static constexpr intptr_t kBlockMask = ~(kBlockSize - 1); -static constexpr intptr_t kBlocksPerPage = kOldPageSize / kBlockSize; - -// A page containing old generation objects. -class OldPage { - public: - enum PageType { kExecutable = 0, kData }; - - OldPage* next() const { return next_; } - void set_next(OldPage* next) { next_ = next; } - - bool Contains(uword addr) const { return memory_->Contains(addr); } - intptr_t AliasOffset() const { return memory_->AliasOffset(); } - - uword object_start() const { return memory_->start() + ObjectStartOffset(); } - uword object_end() const { return object_end_; } - uword used_in_bytes() const { return used_in_bytes_; } - void set_used_in_bytes(uword value) { - ASSERT(Utils::IsAligned(value, kObjectAlignment)); - used_in_bytes_ = value; - } - - ForwardingPage* forwarding_page() const { return forwarding_page_; } - void AllocateForwardingPage(); - - PageType type() const { return type_; } - - bool is_image_page() const { return !memory_->vm_owns_region(); } - - void VisitObjects(ObjectVisitor* visitor) const; - void VisitObjectPointers(ObjectPointerVisitor* visitor) const; - - ObjectPtr FindObject(FindObjectVisitor* visitor) const; - - void WriteProtect(bool read_only); - - static intptr_t ObjectStartOffset() { - return Utils::RoundUp(sizeof(OldPage), kMaxObjectAlignment); - } - - // Warning: This does not work for objects on image pages because image pages - // are not aligned. However, it works for objects on large pages, because - // only one object is allocated per large page. - static OldPage* Of(ObjectPtr obj) { - ASSERT(obj->IsHeapObject()); - ASSERT(obj->IsOldObject()); - return reinterpret_cast<OldPage*>(static_cast<uword>(obj) & kOldPageMask); - } - - // Warning: This does not work for addresses on image pages or on large pages. - static OldPage* Of(uword addr) { - return reinterpret_cast<OldPage*>(addr & kOldPageMask); - } - - // Warning: This does not work for objects on image pages. - static ObjectPtr ToExecutable(ObjectPtr obj) { - OldPage* page = Of(obj); - VirtualMemory* memory = page->memory_; - const intptr_t alias_offset = memory->AliasOffset(); - if (alias_offset == 0) { - return obj; // Not aliased. - } - uword addr = UntaggedObject::ToAddr(obj); - if (memory->Contains(addr)) { - return UntaggedObject::FromAddr(addr + alias_offset); - } - // obj is executable. - ASSERT(memory->ContainsAlias(addr)); - return obj; - } - - // Warning: This does not work for objects on image pages. - static ObjectPtr ToWritable(ObjectPtr obj) { - OldPage* page = Of(obj); - VirtualMemory* memory = page->memory_; - const intptr_t alias_offset = memory->AliasOffset(); - if (alias_offset == 0) { - return obj; // Not aliased. - } - uword addr = UntaggedObject::ToAddr(obj); - if (memory->ContainsAlias(addr)) { - return UntaggedObject::FromAddr(addr - alias_offset); - } - // obj is writable. - ASSERT(memory->Contains(addr)); - return obj; - } - - // 1 card = 128 slots. - static const intptr_t kSlotsPerCardLog2 = 7; - static const intptr_t kBytesPerCardLog2 = - kCompressedWordSizeLog2 + kSlotsPerCardLog2; - - intptr_t card_table_size() const { - return memory_->size() >> kBytesPerCardLog2; - } - - static intptr_t card_table_offset() { - return OFFSET_OF(OldPage, card_table_); - } - - void RememberCard(ObjectPtr const* slot) { - ASSERT(Contains(reinterpret_cast<uword>(slot))); - if (card_table_ == NULL) { - card_table_ = reinterpret_cast<uint8_t*>( - calloc(card_table_size(), sizeof(uint8_t))); - } - intptr_t offset = - reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this); - intptr_t index = offset >> kBytesPerCardLog2; - ASSERT((index >= 0) && (index < card_table_size())); - card_table_[index] = 1; - } - bool IsCardRemembered(ObjectPtr const* slot) { - ASSERT(Contains(reinterpret_cast<uword>(slot))); - if (card_table_ == NULL) { - return false; - } - intptr_t offset = - reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this); - intptr_t index = offset >> kBytesPerCardLog2; - ASSERT((index >= 0) && (index < card_table_size())); - return card_table_[index] != 0; - } -#if defined(DART_COMPRESSED_POINTERS) - void RememberCard(CompressedObjectPtr const* slot) { - ASSERT(Contains(reinterpret_cast<uword>(slot))); - if (card_table_ == NULL) { - card_table_ = reinterpret_cast<uint8_t*>( - calloc(card_table_size(), sizeof(uint8_t))); - } - intptr_t offset = - reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this); - intptr_t index = offset >> kBytesPerCardLog2; - ASSERT((index >= 0) && (index < card_table_size())); - card_table_[index] = 1; - } - bool IsCardRemembered(CompressedObjectPtr const* slot) { - ASSERT(Contains(reinterpret_cast<uword>(slot))); - if (card_table_ == NULL) { - return false; - } - intptr_t offset = - reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this); - intptr_t index = offset >> kBytesPerCardLog2; - ASSERT((index >= 0) && (index < card_table_size())); - return card_table_[index] != 0; - } -#endif - void VisitRememberedCards(ObjectPointerVisitor* visitor); - void ResetProgressBar(); - - private: - void set_object_end(uword value) { - ASSERT((value & kObjectAlignmentMask) == kOldObjectAlignmentOffset); - object_end_ = value; - } - - // Returns NULL on OOM. - static OldPage* Allocate(intptr_t size_in_words, - PageType type, - const char* name); - - // Deallocate the virtual memory backing this page. The page pointer to this - // page becomes immediately inaccessible. - void Deallocate(); - - VirtualMemory* memory_; - OldPage* next_; - uword object_end_; - uword used_in_bytes_; - ForwardingPage* forwarding_page_; - uint8_t* card_table_; // Remembered set, not marking. - RelaxedAtomic<intptr_t> progress_bar_; - PageType type_; - - friend class PageSpace; - friend class GCCompactor; - - DISALLOW_ALLOCATION(); - DISALLOW_IMPLICIT_CONSTRUCTORS(OldPage); -}; - // The history holds the timing information of the last garbage collection // runs. class PageSpaceGarbageCollectionHistory { @@ -328,10 +139,9 @@ ~PageSpace(); uword TryAllocate(intptr_t size, - OldPage::PageType type = OldPage::kData, + Page::PageType type = Page::kData, GrowthPolicy growth_policy = kControlGrowth) { - bool is_protected = - (type == OldPage::kExecutable) && FLAG_write_protect_code; + bool is_protected = (type == Page::kExecutable) && FLAG_write_protect_code; bool is_locked = false; return TryAllocateInternal(size, &freelists_[type], type, growth_policy, is_protected, is_locked); @@ -381,7 +191,7 @@ int64_t ImageInWords() const { int64_t size = 0; MutexLocker ml(&pages_lock_); - for (OldPage* page = image_pages_; page != nullptr; page = page->next()) { + for (Page* page = image_pages_; page != nullptr; page = page->next()) { size += page->memory_->size(); } return size >> kWordSizeLog2; @@ -389,7 +199,7 @@ bool Contains(uword addr) const; bool ContainsUnsafe(uword addr) const; - bool Contains(uword addr, OldPage::PageType type) const; + bool Contains(uword addr, Page::PageType type) const; bool DataContains(uword addr) const; bool IsValidAddress(uword addr) const { return Contains(addr); } @@ -401,8 +211,7 @@ void VisitRememberedCards(ObjectPointerVisitor* visitor) const; void ResetProgressBars() const; - ObjectPtr FindObject(FindObjectVisitor* visitor, - OldPage::PageType type) const; + ObjectPtr FindObject(FindObjectVisitor* visitor, Page::PageType type) const; // Collect the garbage in the page space using mark-sweep or mark-compact. void CollectGarbage(Thread* thread, bool compact, bool finalize); @@ -460,7 +269,7 @@ // Bulk data allocation. FreeList* DataFreeList(intptr_t i = 0) { - return &freelists_[OldPage::kData + i]; + return &freelists_[Page::kData + i]; } void AcquireLock(FreeList* freelist); void ReleaseLock(FreeList* freelist); @@ -470,7 +279,7 @@ GrowthPolicy growth_policy) { bool is_protected = false; bool is_locked = true; - return TryAllocateInternal(size, freelist, OldPage::kData, growth_policy, + return TryAllocateInternal(size, freelist, Page::kData, growth_policy, is_protected, is_locked); } @@ -490,7 +299,7 @@ // Attempt to allocate from bump block rather than normal freelist. uword TryAllocateDataBumpLocked(intptr_t size) { - return TryAllocateDataBumpLocked(&freelists_[OldPage::kData], size); + return TryAllocateDataBumpLocked(&freelists_[Page::kData], size); } uword TryAllocateDataBumpLocked(FreeList* freelist, intptr_t size); DART_FORCE_INLINE @@ -532,36 +341,36 @@ uword TryAllocateInternal(intptr_t size, FreeList* freelist, - OldPage::PageType type, + Page::PageType type, GrowthPolicy growth_policy, bool is_protected, bool is_locked); uword TryAllocateInFreshPage(intptr_t size, FreeList* freelist, - OldPage::PageType type, + Page::PageType type, GrowthPolicy growth_policy, bool is_locked); uword TryAllocateInFreshLargePage(intptr_t size, - OldPage::PageType type, + Page::PageType type, GrowthPolicy growth_policy); // Makes bump block walkable; do not call concurrently with mutator. void MakeIterable() const; - void AddPageLocked(OldPage* page); - void AddLargePageLocked(OldPage* page); - void AddExecPageLocked(OldPage* page); - void RemovePageLocked(OldPage* page, OldPage* previous_page); - void RemoveLargePageLocked(OldPage* page, OldPage* previous_page); - void RemoveExecPageLocked(OldPage* page, OldPage* previous_page); + void AddPageLocked(Page* page); + void AddLargePageLocked(Page* page); + void AddExecPageLocked(Page* page); + void RemovePageLocked(Page* page, Page* previous_page); + void RemoveLargePageLocked(Page* page, Page* previous_page); + void RemoveExecPageLocked(Page* page, Page* previous_page); - OldPage* AllocatePage(OldPage::PageType type, bool link = true); - OldPage* AllocateLargePage(intptr_t size, OldPage::PageType type); + Page* AllocatePage(Page::PageType type, bool link = true); + Page* AllocateLargePage(intptr_t size, Page::PageType type); - void TruncateLargePage(OldPage* page, intptr_t new_object_size_in_bytes); - void FreePage(OldPage* page, OldPage* previous_page); - void FreeLargePage(OldPage* page, OldPage* previous_page); - void FreePages(OldPage* pages); + void TruncateLargePage(Page* page, intptr_t new_object_size_in_bytes); + void FreePage(Page* page, Page* previous_page); + void FreeLargePage(Page* page, Page* previous_page); + void FreePages(Page* pages, bool can_use_cache); void CollectGarbageHelper(Thread* thread, bool compact, bool finalize); void SweepLarge(); @@ -584,9 +393,9 @@ Heap* const heap_; - // One list for executable pages at freelists_[OldPage::kExecutable]. + // One list for executable pages at freelists_[Page::kExecutable]. // FLAG_scavenger_tasks count of lists for data pages starting at - // freelists_[OldPage::kData]. The sweeper inserts into the data page + // freelists_[Page::kData]. The sweeper inserts into the data page // freelists round-robin. The scavenger workers each use one of the data // page freelists without locking. const intptr_t num_freelists_; @@ -596,15 +405,15 @@ // Use ExclusivePageIterator for safe access to these. mutable Mutex pages_lock_; - OldPage* pages_ = nullptr; - OldPage* pages_tail_ = nullptr; - OldPage* exec_pages_ = nullptr; - OldPage* exec_pages_tail_ = nullptr; - OldPage* large_pages_ = nullptr; - OldPage* large_pages_tail_ = nullptr; - OldPage* image_pages_ = nullptr; - OldPage* sweep_regular_ = nullptr; - OldPage* sweep_large_ = nullptr; + Page* pages_ = nullptr; + Page* pages_tail_ = nullptr; + Page* exec_pages_ = nullptr; + Page* exec_pages_tail_ = nullptr; + Page* large_pages_ = nullptr; + Page* large_pages_tail_ = nullptr; + Page* image_pages_ = nullptr; + Page* sweep_regular_ = nullptr; + Page* sweep_large_ = nullptr; // Various sizes being tracked for this generation. intptr_t max_capacity_in_words_;
diff --git a/runtime/vm/heap/scavenger.cc b/runtime/vm/heap/scavenger.cc index 959da6f..6ac1820 100644 --- a/runtime/vm/heap/scavenger.cc +++ b/runtime/vm/heap/scavenger.cc
@@ -237,7 +237,7 @@ ASSERT((obj == nullptr) || obj->IsOldObject()); visiting_old_object_ = obj; if (obj != nullptr) { - // Card update happens in OldPage::VisitRememberedCards. + // Card update happens in Page::VisitRememberedCards. ASSERT(!obj->untag()->IsCardRemembered()); } } @@ -300,7 +300,7 @@ if (!scavenger_->abort_) { ASSERT(!HasWork()); - for (NewPage* page = head_; page != nullptr; page = page->next()) { + for (Page* page = head_; page != nullptr; page = page->next()) { ASSERT(page->IsResolved()); page->RecordSurvivors(); } @@ -320,8 +320,12 @@ delayed_.Release(); } - NewPage* head() const { return head_; } - NewPage* tail() const { return tail_; } + Page* head() const { + return head_; + } + Page* tail() const { + return tail_; + } static bool ForwardOrSetNullIfCollected(uword heap_base, CompressedObjectPtr* ptr_address); @@ -403,7 +407,7 @@ intptr_t size = raw_obj->untag()->HeapSize(header); uword new_addr = 0; // Check whether object should be promoted. - if (!NewPage::Of(raw_obj)->IsSurvivor(raw_addr)) { + if (!Page::Of(raw_obj)->IsSurvivor(raw_addr)) { // Not a survivor of a previous scavenge. Just copy the object into the // to space. new_addr = TryAllocateCopy(size); @@ -542,9 +546,9 @@ PromotionWorkList promoted_list_; GCLinkedLists delayed_; - NewPage* head_ = nullptr; - NewPage* tail_ = nullptr; // Allocating from here. - NewPage* scan_ = nullptr; // Resolving from here. + Page* head_ = nullptr; + Page* tail_ = nullptr; // Allocating from here. + Page* scan_ = nullptr; // Resolving from here. template <typename GCVisitorType> friend void MournFinalized(GCVisitorType* visitor); @@ -674,121 +678,23 @@ : max_capacity_in_words_(max_capacity_in_words), head_(nullptr) {} SemiSpace::~SemiSpace() { - NewPage* page = head_; + Page* page = head_; while (page != nullptr) { - NewPage* next = page->next(); - page->Deallocate(); + Page* next = page->next(); + page->Deallocate(/*can_use_cache*/ true); page = next; } } -// TODO(rmacnak): Unify this with old-space pages, and possibly zone segments. -// This cache needs to be at least as big as FLAG_new_gen_semi_max_size or -// munmap will noticably impact performance. -static constexpr intptr_t kPageCacheCapacity = 8 * kWordSize; -static Mutex* page_cache_mutex = nullptr; -static VirtualMemory* page_cache[kPageCacheCapacity] = {nullptr}; -static intptr_t page_cache_size = 0; - -void SemiSpace::Init() { - ASSERT(page_cache_mutex == nullptr); - page_cache_mutex = new Mutex(NOT_IN_PRODUCT("page_cache_mutex")); -} - -void SemiSpace::ClearCache() { - MutexLocker ml(page_cache_mutex); - ASSERT(page_cache_size >= 0); - ASSERT(page_cache_size <= kPageCacheCapacity); - while (page_cache_size > 0) { - delete page_cache[--page_cache_size]; - } -} - -void SemiSpace::Cleanup() { - ClearCache(); - delete page_cache_mutex; - page_cache_mutex = nullptr; -} - -intptr_t SemiSpace::CachedSize() { - MutexLocker ml(page_cache_mutex); - return page_cache_size * kNewPageSize; -} - -NewPage* NewPage::Allocate() { - const intptr_t size = kNewPageSize; - VirtualMemory* memory = nullptr; - { - MutexLocker ml(page_cache_mutex); - ASSERT(page_cache_size >= 0); - ASSERT(page_cache_size <= kPageCacheCapacity); - if (page_cache_size > 0) { - memory = page_cache[--page_cache_size]; - } - } - if (memory == nullptr) { - const intptr_t alignment = kNewPageSize; - const bool is_executable = false; - const bool compressed = true; - const char* const name = Heap::RegionName(Heap::kNew); - memory = VirtualMemory::AllocateAligned(size, alignment, is_executable, - compressed, name); - } - if (memory == nullptr) { - return nullptr; // Out of memory. - } - -#if defined(DEBUG) - memset(memory->address(), Heap::kZapByte, size); -#endif - // Initialized by generated code. - MSAN_UNPOISON(memory->address(), size); - - NewPage* result = reinterpret_cast<NewPage*>(memory->address()); - result->memory_ = memory; - result->next_ = nullptr; - result->owner_ = nullptr; - uword top = result->object_start(); - result->top_ = top; - result->end_ = memory->end() - kNewObjectAlignmentOffset; - result->survivor_end_ = top; - result->resolved_top_ = top; - - LSAN_REGISTER_ROOT_REGION(result, sizeof(*result)); - - return result; -} - -void NewPage::Deallocate() { - LSAN_UNREGISTER_ROOT_REGION(this, sizeof(*this)); - - VirtualMemory* memory = memory_; - { - MutexLocker ml(page_cache_mutex); - ASSERT(page_cache_size >= 0); - ASSERT(page_cache_size <= kPageCacheCapacity); - if (page_cache_size < kPageCacheCapacity) { - intptr_t size = memory->size(); -#if defined(DEBUG) - memset(memory->address(), Heap::kZapByte, size); -#endif - MSAN_POISON(memory->address(), size); - page_cache[page_cache_size++] = memory; - memory = nullptr; - } - } - delete memory; -} - -NewPage* SemiSpace::TryAllocatePageLocked(bool link) { +Page* SemiSpace::TryAllocatePageLocked(bool link) { if (capacity_in_words_ >= max_capacity_in_words_) { return nullptr; // Full. } - NewPage* page = NewPage::Allocate(); + Page* page = Page::Allocate(kPageSize, Page::kNew, /*can_use_cache*/ true); if (page == nullptr) { return nullptr; // Out of memory; } - capacity_in_words_ += kNewPageSizeInWords; + capacity_in_words_ += kPageSizeInWords; if (link) { if (head_ == nullptr) { head_ = tail_ = page; @@ -801,19 +707,19 @@ } bool SemiSpace::Contains(uword addr) const { - for (NewPage* page = head_; page != nullptr; page = page->next()) { + for (Page* page = head_; page != nullptr; page = page->next()) { if (page->Contains(addr)) return true; } return false; } void SemiSpace::WriteProtect(bool read_only) { - for (NewPage* page = head_; page != nullptr; page = page->next()) { + for (Page* page = head_; page != nullptr; page = page->next()) { page->WriteProtect(read_only); } } -void SemiSpace::AddList(NewPage* head, NewPage* tail) { +void SemiSpace::AddList(Page* head, Page* tail) { if (head == nullptr) { return; } @@ -868,7 +774,7 @@ GCReason reason) const { bool grow = false; if (2 * heap_->isolate_group()->MutatorCount() > - (old_size_in_words / kNewPageSizeInWords)) { + (old_size_in_words / kPageSizeInWords)) { // Not enough TLABs to give two to each mutator. grow = true; } @@ -957,7 +863,7 @@ ObjectPtr raw_obj = *ptr; if (raw_obj->IsHeapObject() && raw_obj->IsNewObject()) { if (is_card_remembered_) { - if (!OldPage::Of(visiting_)->IsCardRemembered(ptr)) { + if (!Page::Of(visiting_)->IsCardRemembered(ptr)) { FATAL3( "Old object %#" Px " references new object %#" Px ", but the " @@ -988,7 +894,7 @@ ObjectPtr raw_obj = ptr->Decompress(heap_base); if (raw_obj->IsHeapObject() && raw_obj->IsNewObject()) { if (is_card_remembered_) { - if (!OldPage::Of(visiting_)->IsCardRemembered(ptr)) { + if (!Page::Of(visiting_)->IsCardRemembered(ptr)) { FATAL3( "Old object %#" Px " references new object %#" Px ", but the " @@ -1350,7 +1256,7 @@ } scan_->resolved_top_ = resolved_top; - NewPage* next = scan_->next(); + Page* next = scan_->next(); if (next == nullptr) { // Don't update scan_. More objects may yet be copied to this TLAB. return; @@ -1633,7 +1539,7 @@ ASSERT(Thread::Current()->IsAtSafepoint() || (Thread::Current()->task_kind() == Thread::kMarkerTask) || (Thread::Current()->task_kind() == Thread::kCompactorTask)); - for (NewPage* page = to_->head(); page != nullptr; page = page->next()) { + for (Page* page = to_->head(); page != nullptr; page = page->next()) { page->VisitObjectPointers(visitor); } } @@ -1641,20 +1547,20 @@ void Scavenger::VisitObjects(ObjectVisitor* visitor) const { ASSERT(Thread::Current()->IsAtSafepoint() || (Thread::Current()->task_kind() == Thread::kMarkerTask)); - for (NewPage* page = to_->head(); page != nullptr; page = page->next()) { + for (Page* page = to_->head(); page != nullptr; page = page->next()) { page->VisitObjects(visitor); } } void Scavenger::AddRegionsToObjectSet(ObjectSet* set) const { - for (NewPage* page = to_->head(); page != nullptr; page = page->next()) { + for (Page* page = to_->head(); page != nullptr; page = page->next()) { set->AddRegion(page->start(), page->end()); } } ObjectPtr Scavenger::FindObject(FindObjectVisitor* visitor) { ASSERT(!scavenging_); - for (NewPage* page = to_->head(); page != nullptr; page = page->next()) { + for (Page* page = to_->head(); page != nullptr; page = page->next()) { uword cur = page->object_start(); if (!visitor->VisitRange(cur, page->object_end())) continue; while (cur < page->object_end()) { @@ -1681,20 +1587,21 @@ if (can_safepoint && !thread->force_growth()) { ASSERT(thread->no_safepoint_scope_depth() == 0); - heap_->CheckConcurrentMarking(thread, GCReason::kNewSpace, kNewPageSize); + heap_->CheckConcurrentMarking(thread, GCReason::kNewSpace, kPageSize); } MutexLocker ml(&space_lock_); - for (NewPage* page = to_->head(); page != nullptr; page = page->next()) { + for (Page* page = to_->head(); page != nullptr; page = page->next()) { if (page->owner() != nullptr) continue; - intptr_t available = page->end() - page->object_end(); + intptr_t available = + (page->end() - kAllocationRedZoneSize) - page->object_end(); if (available >= min_size) { page->Acquire(thread); return; } } - NewPage* page = to_->TryAllocatePageLocked(true); + Page* page = to_->TryAllocatePageLocked(true); if (page == nullptr) { return; } @@ -1716,7 +1623,7 @@ void Scavenger::AbandonRemainingTLAB(Thread* thread) { if (thread->top() == 0) return; - NewPage* page = NewPage::Of(thread->top() - 1); + Page* page = Page::Of(thread->top() - 1); { MutexLocker ml(&space_lock_); page->Release(thread); @@ -1726,7 +1633,7 @@ template <bool parallel> uword ScavengerVisitorBase<parallel>::TryAllocateCopySlow(intptr_t size) { - NewPage* page; + Page* page; { MutexLocker ml(&scavenger_->space_lock_); page = scavenger_->to_->TryAllocatePageLocked(false); @@ -1774,7 +1681,7 @@ intptr_t abandoned_bytes = 0; // TODO(rmacnak): Count fragmentation? SpaceUsage usage_before = GetCurrentUsage(); intptr_t promo_candidate_words = 0; - for (NewPage* page = to_->head(); page != nullptr; page = page->next()) { + for (Page* page = to_->head(); page != nullptr; page = page->next()) { page->Release(); if (early_tenure_) { page->EarlyTenure(); @@ -1920,7 +1827,7 @@ }; ReverseFromForwardingVisitor visitor; - for (NewPage* page = (*from)->head(); page != nullptr; page = page->next()) { + for (Page* page = (*from)->head(); page != nullptr; page = page->next()) { page->VisitObjects(&visitor); }
diff --git a/runtime/vm/heap/scavenger.h b/runtime/vm/heap/scavenger.h index f5215ac..a54680d 100644 --- a/runtime/vm/heap/scavenger.h +++ b/runtime/vm/heap/scavenger.h
@@ -11,6 +11,7 @@ #include "vm/dart.h" #include "vm/flags.h" #include "vm/globals.h" +#include "vm/heap/page.h" #include "vm/heap/spaces.h" #include "vm/isolate.h" #include "vm/lockers.h" @@ -29,166 +30,19 @@ template <bool parallel> class ScavengerVisitorBase; -static constexpr intptr_t kNewPageSize = 512 * KB; -static constexpr intptr_t kNewPageSizeInWords = kNewPageSize / kWordSize; -static constexpr intptr_t kNewPageMask = ~(kNewPageSize - 1); - -// Simplify initialization in allocation stubs by ensuring it is safe -// to overshoot the object end by up to kAllocationRedZoneSize. (Just as the -// stack red zone allows one to overshoot the stack pointer.) -static constexpr intptr_t kAllocationRedZoneSize = kObjectAlignment; - -// A page containing new generation objects. -class NewPage { - public: - static NewPage* Allocate(); - void Deallocate(); - - uword start() const { return memory_->start(); } - uword end() const { return memory_->end() - kAllocationRedZoneSize; } - bool Contains(uword addr) const { return memory_->Contains(addr); } - void WriteProtect(bool read_only) { - memory_->Protect(read_only ? VirtualMemory::kReadOnly - : VirtualMemory::kReadWrite); - } - - NewPage* next() const { return next_; } - void set_next(NewPage* next) { next_ = next; } - - Thread* owner() const { return owner_; } - - uword object_start() const { return start() + ObjectStartOffset(); } - uword object_end() const { return owner_ != nullptr ? owner_->top() : top_; } - intptr_t used() const { return object_end() - object_start(); } - void VisitObjects(ObjectVisitor* visitor) const { - uword addr = object_start(); - uword end = object_end(); - while (addr < end) { - ObjectPtr obj = UntaggedObject::FromAddr(addr); - visitor->VisitObject(obj); - addr += obj->untag()->HeapSize(); - } - } - void VisitObjectPointers(ObjectPointerVisitor* visitor) const { - uword addr = object_start(); - uword end = object_end(); - while (addr < end) { - ObjectPtr obj = UntaggedObject::FromAddr(addr); - intptr_t size = obj->untag()->VisitPointers(visitor); - addr += size; - } - } - - static intptr_t ObjectStartOffset() { - return Utils::RoundUp(sizeof(NewPage), kObjectAlignment) + - kNewObjectAlignmentOffset; - } - - static NewPage* Of(ObjectPtr obj) { - ASSERT(obj->IsHeapObject()); - ASSERT(obj->IsNewObject()); - return Of(static_cast<uword>(obj)); - } - static NewPage* Of(uword addr) { - return reinterpret_cast<NewPage*>(addr & kNewPageMask); - } - - // Remember the limit to which objects have been copied. - void RecordSurvivors() { survivor_end_ = object_end(); } - - // Move survivor end to the end of the to_ space, making all surviving - // objects candidates for promotion next time. - void EarlyTenure() { survivor_end_ = end_; } - - uword promo_candidate_words() const { - return (survivor_end_ - object_start()) / kWordSize; - } - - void Acquire(Thread* thread) { - ASSERT(owner_ == nullptr); - owner_ = thread; - thread->set_top(top_); - thread->set_end(end_); - } - void Release(Thread* thread) { - ASSERT(owner_ == thread); - owner_ = nullptr; - top_ = thread->top(); - thread->set_top(0); - thread->set_end(0); - } - void Release() { - if (owner_ != nullptr) { - Release(owner_); - } - } - - uword TryAllocateGC(intptr_t size) { - ASSERT(owner_ == nullptr); - uword result = top_; - uword new_top = result + size; - if (LIKELY(new_top <= end_)) { - top_ = new_top; - return result; - } - return 0; - } - - void Unallocate(uword addr, intptr_t size) { - ASSERT((addr + size) == top_); - top_ -= size; - } - - bool IsSurvivor(uword raw_addr) const { return raw_addr < survivor_end_; } - bool IsResolved() const { return top_ == resolved_top_; } - - private: - VirtualMemory* memory_; - NewPage* next_; - - // The thread using this page for allocation, otherwise NULL. - Thread* owner_; - - // The address of the next allocation. If owner is non-NULL, this value is - // stale and the current value is at owner->top_. Called "NEXT" in the - // original Cheney paper. - uword top_; - - // The address after the last allocatable byte in this page. - uword end_; - - // Objects below this address have survived a scavenge. - uword survivor_end_; - - // A pointer to the first unprocessed object. Resolution completes when this - // value meets the allocation top. Called "SCAN" in the original Cheney paper. - uword resolved_top_; - - template <bool> - friend class ScavengerVisitorBase; - - DISALLOW_ALLOCATION(); - DISALLOW_IMPLICIT_CONSTRUCTORS(NewPage); -}; - class SemiSpace { public: - static void Init(); - static void ClearCache(); - static void Cleanup(); - static intptr_t CachedSize(); - explicit SemiSpace(intptr_t max_capacity_in_words); ~SemiSpace(); - NewPage* TryAllocatePageLocked(bool link); + Page* TryAllocatePageLocked(bool link); bool Contains(uword addr) const; void WriteProtect(bool read_only); intptr_t used_in_words() const { intptr_t size = 0; - for (const NewPage* p = head_; p != nullptr; p = p->next()) { + for (const Page* p = head_; p != nullptr; p = p->next()) { size += p->used(); } return size >> kWordSizeLog2; @@ -196,9 +50,9 @@ intptr_t capacity_in_words() const { return capacity_in_words_; } intptr_t max_capacity_in_words() const { return max_capacity_in_words_; } - NewPage* head() const { return head_; } + Page* head() const { return head_; } - void AddList(NewPage* head, NewPage* tail); + void AddList(Page* head, Page* tail); private: // Size of NewPages in this semi-space. @@ -207,8 +61,8 @@ // Size of NewPages before we trigger a scavenge. intptr_t max_capacity_in_words_; - NewPage* head_ = nullptr; - NewPage* tail_ = nullptr; + Page* head_ = nullptr; + Page* tail_ = nullptr; }; // Statistics for a particular scavenge. @@ -372,7 +226,9 @@ return max_pool_size > 0 ? max_pool_size : 1; } - NewPage* head() const { return to_->head(); } + Page* head() const { + return to_->head(); + } private: // Ids for time and data records in Heap::GCStats.
diff --git a/runtime/vm/heap/sweeper.cc b/runtime/vm/heap/sweeper.cc index 11900bb..faf54b8 100644 --- a/runtime/vm/heap/sweeper.cc +++ b/runtime/vm/heap/sweeper.cc
@@ -15,20 +15,20 @@ namespace dart { -bool GCSweeper::SweepPage(OldPage* page, FreeList* freelist, bool locked) { +bool GCSweeper::SweepPage(Page* page, FreeList* freelist, bool locked) { ASSERT(!page->is_image_page()); // Keep track whether this page is still in use. intptr_t used_in_bytes = 0; - bool is_executable = (page->type() == OldPage::kExecutable); + bool is_executable = (page->type() == Page::kExecutable); uword start = page->object_start(); uword end = page->object_end(); uword current = start; while (current < end) { ObjectPtr raw_obj = UntaggedObject::FromAddr(current); - ASSERT(OldPage::Of(raw_obj) == page); + ASSERT(Page::Of(raw_obj) == page); // These acquire operations balance release operations in array // truncaton, ensuring the writes creating the filler object are ordered // before the writes inserting the filler object into the freelist. @@ -76,16 +76,15 @@ } ASSERT(current == end); - page->set_used_in_bytes(used_in_bytes); return used_in_bytes != 0; // In use. } -intptr_t GCSweeper::SweepLargePage(OldPage* page) { +intptr_t GCSweeper::SweepLargePage(Page* page) { ASSERT(!page->is_image_page()); intptr_t words_to_end = 0; ObjectPtr raw_obj = UntaggedObject::FromAddr(page->object_start()); - ASSERT(OldPage::Of(raw_obj) == page); + ASSERT(Page::Of(raw_obj) == page); if (raw_obj->untag()->IsMarked()) { raw_obj->untag()->ClearMarkBit(); words_to_end = (raw_obj->untag()->HeapSize() >> kWordSizeLog2);
diff --git a/runtime/vm/heap/sweeper.h b/runtime/vm/heap/sweeper.h index 8b781d4..99e22e2 100644 --- a/runtime/vm/heap/sweeper.h +++ b/runtime/vm/heap/sweeper.h
@@ -12,7 +12,7 @@ // Forward declarations. class FreeList; class Heap; -class OldPage; +class Page; class IsolateGroup; class PageSpace; @@ -28,11 +28,11 @@ // pre-locked is indicated by the locked parameter. // Returns true if the page is in use. Freelist is untouched if page is not // in use. - bool SweepPage(OldPage* page, FreeList* freelist, bool locked); + bool SweepPage(Page* page, FreeList* freelist, bool locked); // Returns the number of words from page->object_start() to the end of the // last marked object. - intptr_t SweepLargePage(OldPage* page); + intptr_t SweepLargePage(Page* page); // Sweep the large and regular sized data pages. static void SweepConcurrent(IsolateGroup* isolate_group);
diff --git a/runtime/vm/heap/verifier.cc b/runtime/vm/heap/verifier.cc index 1a726dd..148c901 100644 --- a/runtime/vm/heap/verifier.cc +++ b/runtime/vm/heap/verifier.cc
@@ -50,7 +50,7 @@ if (raw_obj->IsHeapObject()) { if (!allocated_set_->Contains(raw_obj)) { if (raw_obj->IsInstructions() && - allocated_set_->Contains(OldPage::ToWritable(raw_obj))) { + allocated_set_->Contains(Page::ToWritable(raw_obj))) { continue; } uword raw_addr = UntaggedObject::ToAddr(raw_obj); @@ -68,7 +68,7 @@ if (raw_obj->IsHeapObject()) { if (!allocated_set_->Contains(raw_obj)) { if (raw_obj->IsInstructions() && - allocated_set_->Contains(OldPage::ToWritable(raw_obj))) { + allocated_set_->Contains(Page::ToWritable(raw_obj))) { continue; } uword raw_addr = UntaggedObject::ToAddr(raw_obj);
diff --git a/runtime/vm/image_snapshot.cc b/runtime/vm/image_snapshot.cc index 6cf7e67..01cd7af 100644 --- a/runtime/vm/image_snapshot.cc +++ b/runtime/vm/image_snapshot.cc
@@ -685,7 +685,9 @@ const bool bare_instruction_payloads = FLAG_precompiled_mode; // Start snapshot at page boundary. - if (!EnterSection(ProgramSection::Text, vm, ImageWriter::kTextAlignment)) { + intptr_t alignment_padding = 0; + if (!EnterSection(ProgramSection::Text, vm, ImageWriter::kTextAlignment, + &alignment_padding)) { return; } @@ -698,7 +700,7 @@ #endif // This head also provides the gap to make the instructions snapshot - // look like a OldPage. + // look like a Page. const intptr_t image_size = Utils::RoundUp( next_text_offset_, compiler::target::ObjectAlignment::kObjectAlignment); text_offset += WriteTargetWord(image_size); @@ -720,7 +722,8 @@ if (profile_writer_ != nullptr) { profile_writer_->SetObjectTypeAndName(parent_id, image_type_, instructions_symbol); - profile_writer_->AttributeBytesTo(parent_id, Image::kHeaderSize); + profile_writer_->AttributeBytesTo( + parent_id, ImageWriter::kTextAlignment + alignment_padding); profile_writer_->AddRoot(parent_id); } @@ -1319,7 +1322,8 @@ bool AssemblyImageWriter::EnterSection(ProgramSection section, bool vm, - intptr_t alignment) { + intptr_t alignment, + intptr_t* alignment_padding) { ASSERT(FLAG_precompiled_mode); ASSERT(current_section_symbol_ == nullptr); ASSERT(current_symbols_ == nullptr); @@ -1363,7 +1367,10 @@ if (global_symbol) { assembly_stream_->Printf(".globl %s\n", current_section_symbol_); } - Align(alignment); + intptr_t padding = Align(alignment); + if (alignment_padding != nullptr) { + *alignment_padding = padding; + } assembly_stream_->Printf("%s:\n", current_section_symbol_); return true; } @@ -1652,7 +1659,8 @@ bool BlobImageWriter::EnterSection(ProgramSection section, bool vm, - intptr_t alignment) { + intptr_t alignment, + intptr_t* alignment_padding) { #if defined(DART_PRECOMPILER) ASSERT_EQUAL(elf_ != nullptr, FLAG_precompiled_mode); ASSERT(current_relocations_ == nullptr); @@ -1691,7 +1699,10 @@ return false; } current_section_symbol_ = SectionSymbol(section, vm); - current_section_stream_->Align(alignment); + intptr_t padding = current_section_stream_->Align(alignment); + if (alignment_padding != nullptr) { + *alignment_padding = padding; + } return true; } @@ -1769,8 +1780,8 @@ instructions_image_(ASSERT_NOTNULL(instructions_image)) {} ApiErrorPtr ImageReader::VerifyAlignment() const { - if (!Utils::IsAligned(data_image_, kObjectAlignment) || - !Utils::IsAligned(instructions_image_, kMaxObjectAlignment)) { + if (!Utils::IsAligned(data_image_, kObjectStartAlignment) || + !Utils::IsAligned(instructions_image_, kObjectStartAlignment)) { return ApiError::New( String::Handle(String::New("Snapshot is misaligned", Heap::kOld)), Heap::kOld);
diff --git a/runtime/vm/image_snapshot.h b/runtime/vm/image_snapshot.h index d7094d0..abbcabf 100644 --- a/runtime/vm/image_snapshot.h +++ b/runtime/vm/image_snapshot.h
@@ -44,7 +44,7 @@ : raw_memory_(raw_memory), snapshot_size_(FieldValue(raw_memory, HeaderField::ImageSize)), extra_info_(ExtraInfo(raw_memory_, snapshot_size_)) { - ASSERT(Utils::IsAligned(raw_memory, kMaxObjectAlignment)); + ASSERT(Utils::IsAligned(raw_memory, kObjectStartAlignment)); } // Even though an Image is read-only memory, we must return a void* here. @@ -117,12 +117,12 @@ // // Note: Image::kHeaderSize is _not_ an architecture-dependent constant, // and so there is no compiler::target::Image::kHeaderSize. - static constexpr intptr_t kHeaderSize = kMaxObjectAlignment; + static constexpr intptr_t kHeaderSize = kObjectStartAlignment; // Explicitly double-checking kHeaderSize is never changed. Increasing the // Image header size would mean objects would not start at a place expected - // by parts of the VM (like the GC) that use Image pages as HeapPages. - static_assert(kHeaderSize == kMaxObjectAlignment, - "Image page cannot be used as HeapPage"); + // by parts of the VM (like the GC) that use Image pages as Pages. + static_assert(kHeaderSize == kObjectStartAlignment, + "Image page cannot be used as Page"); // Make sure that the number of fields in the Image header fit both on the // host and target architectures. static_assert(kHeaderFields * kWordSize <= kHeaderSize, @@ -243,10 +243,10 @@ // BSS sections contain word-sized data. static constexpr intptr_t kBssAlignment = compiler::target::kWordSize; // ROData sections contain objects wrapped in an Image object. - static constexpr intptr_t kRODataAlignment = kMaxObjectAlignment; + static constexpr intptr_t kRODataAlignment = kObjectStartAlignment; // Text sections contain objects (even in bare instructions mode) wrapped // in an Image object. - static constexpr intptr_t kTextAlignment = kMaxObjectAlignment; + static constexpr intptr_t kTextAlignment = kObjectStartAlignment; void ResetOffsets() { next_data_offset_ = Image::kHeaderSize; @@ -425,7 +425,8 @@ // this section should not be written. virtual bool EnterSection(ProgramSection name, bool vm, - intptr_t alignment) = 0; + intptr_t alignment, + intptr_t* alignment_padding = nullptr) = 0; // Marks the exit from a particular ProgramSection, allowing subclasses to // do any post-writing work. virtual void ExitSection(ProgramSection name, bool vm, intptr_t size) = 0; @@ -604,7 +605,8 @@ virtual bool EnterSection(ProgramSection section, bool vm, - intptr_t alignment); + intptr_t alignment, + intptr_t* alignment_padding = nullptr); virtual void ExitSection(ProgramSection name, bool vm, intptr_t size); virtual intptr_t WriteTargetWord(word value); virtual intptr_t WriteBytes(const void* bytes, intptr_t size); @@ -655,7 +657,8 @@ virtual bool EnterSection(ProgramSection section, bool vm, - intptr_t alignment); + intptr_t alignment, + intptr_t* alignment_padding = nullptr); virtual void ExitSection(ProgramSection name, bool vm, intptr_t size); virtual intptr_t WriteTargetWord(word value); virtual intptr_t WriteBytes(const void* bytes, intptr_t size);
diff --git a/runtime/vm/message.h b/runtime/vm/message.h index 46064a9..31bdb4a 100644 --- a/runtime/vm/message.h +++ b/runtime/vm/message.h
@@ -21,9 +21,6 @@ class JSONStream; class PersistentHandle; -class OldPage; -class WeakTable; -class FreeList; class Message { public:
diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index 2123257..b77df2f 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc
@@ -17483,7 +17483,7 @@ if (FLAG_write_protect_code) { uword address = UntaggedObject::ToAddr(instrs.ptr()); // Check if a dual mapping exists. - instrs = Instructions::RawCast(OldPage::ToExecutable(instrs.ptr())); + instrs = Instructions::RawCast(Page::ToExecutable(instrs.ptr())); uword exec_address = UntaggedObject::ToAddr(instrs.ptr()); const bool use_dual_mapping = exec_address != address; ASSERT(use_dual_mapping == FLAG_dual_map_code);
diff --git a/runtime/vm/object.h b/runtime/vm/object.h index aaf321b..a3ab391 100644 --- a/runtime/vm/object.h +++ b/runtime/vm/object.h
@@ -5450,7 +5450,7 @@ static const intptr_t kMaxElements = (kMaxInt32 - (sizeof(UntaggedInstructions) + sizeof(UntaggedObject) + - (2 * kMaxObjectAlignment))); + (2 * kObjectStartAlignment))); // Currently, we align bare instruction payloads on 4 byte boundaries. //
diff --git a/runtime/vm/object_graph.cc b/runtime/vm/object_graph.cc index da3b779..33824c1 100644 --- a/runtime/vm/object_graph.cc +++ b/runtime/vm/object_graph.cc
@@ -706,7 +706,7 @@ return visitor.length(); } -// Each OldPage is divided into blocks of size kBlockSize. Each object belongs +// Each Page is divided into blocks of size kBlockSize. Each object belongs // to the block containing its header word. // When generating a heap snapshot, we assign objects sequential ids in heap // iteration order. A bitvector is computed that indicates the number of objects @@ -763,7 +763,7 @@ } CountingBlock* BlockFor(uword addr) { - intptr_t page_offset = addr & ~kOldPageMask; + intptr_t page_offset = addr & ~kPageMask; intptr_t block_number = page_offset / kBlockSize; ASSERT(block_number >= 0); ASSERT(block_number <= kBlocksPerPage); @@ -816,7 +816,7 @@ image_page_ranges_[i].size = 0; } intptr_t next_offset = 0; - OldPage* image_page = + Page* image_page = Dart::vm_isolate_group()->heap()->old_space()->image_pages_; while (image_page != NULL) { RELEASE_ASSERT(next_offset <= kMaxImagePages); @@ -836,7 +836,7 @@ next_offset++; } - OldPage* page = isolate_group()->heap()->old_space()->pages_; + Page* page = isolate_group()->heap()->old_space()->pages_; while (page != NULL) { page->forwarding_page(); CountingPage* counting_page = @@ -860,7 +860,7 @@ CountingPage* HeapSnapshotWriter::FindCountingPage(ObjectPtr obj) const { if (obj->IsOldObject() && !OnImagePage(obj)) { // On a regular or large page. - OldPage* page = OldPage::Of(obj); + Page* page = Page::Of(obj); return reinterpret_cast<CountingPage*>(page->forwarding_page()); } @@ -888,7 +888,7 @@ if (FLAG_write_protect_code && obj->IsInstructions() && !OnImagePage(obj)) { // A non-writable alias mapping may exist for instruction pages. - obj = OldPage::ToWritable(obj); + obj = Page::ToWritable(obj); } CountingPage* counting_page = FindCountingPage(obj);
diff --git a/runtime/vm/object_test.cc b/runtime/vm/object_test.cc index 3cc7a2b..5ced93d 100644 --- a/runtime/vm/object_test.cc +++ b/runtime/vm/object_test.cc
@@ -2759,7 +2759,7 @@ Object::Handle(DartEntry::InvokeFunction(function, Array::empty_array())); EXPECT_EQ(1, Smi::Cast(result).Value()); // Switch to the writeable but non-executable view of the instructions. - instructions ^= OldPage::ToWritable(instructions.ptr()); + instructions ^= Page::ToWritable(instructions.ptr()); payload_start = instructions.PayloadStart(); EXPECT_EQ(instructions.ptr(), Instructions::FromPayloadStart(payload_start)); // Hook up Code and Instructions objects.
diff --git a/runtime/vm/pointer_tagging.h b/runtime/vm/pointer_tagging.h index 90dd053..6838506 100644 --- a/runtime/vm/pointer_tagging.h +++ b/runtime/vm/pointer_tagging.h
@@ -71,9 +71,10 @@ static constexpr intptr_t kFalseOffsetFromNull = HostObjectAlignment::kFalseOffsetFromNull; -// The largest value of kObjectAlignment across all configurations. -static constexpr intptr_t kMaxObjectAlignment = 16; -COMPILE_ASSERT(kMaxObjectAlignment >= kObjectAlignment); +static constexpr intptr_t kObjectStartAlignment = 64; +COMPILE_ASSERT(kObjectStartAlignment >= kObjectAlignment); +COMPILE_ASSERT(kObjectStartAlignment >= 2 * kBoolValueMask); +COMPILE_ASSERT(kObjectStartAlignment >= 2 * kBoolVsNullMask); // On all targets heap pointers are tagged by set least significant bit. //
diff --git a/runtime/vm/raw_object.cc b/runtime/vm/raw_object.cc index f8f54c9..4c26e8d 100644 --- a/runtime/vm/raw_object.cc +++ b/runtime/vm/raw_object.cc
@@ -760,12 +760,12 @@ } void UntaggedObject::RememberCard(ObjectPtr const* slot) { - OldPage::Of(static_cast<ObjectPtr>(this))->RememberCard(slot); + Page::Of(static_cast<ObjectPtr>(this))->RememberCard(slot); } #if defined(DART_COMPRESSED_POINTERS) void UntaggedObject::RememberCard(CompressedObjectPtr const* slot) { - OldPage::Of(static_cast<ObjectPtr>(this))->RememberCard(slot); + Page::Of(static_cast<ObjectPtr>(this))->RememberCard(slot); } #endif @@ -777,7 +777,7 @@ ObjectPtr object = static_cast<ObjectPtr>(object_in); ASSERT(object->IsOldObject()); ASSERT(object->untag()->IsCardRemembered()); - OldPage::Of(object)->RememberCard(slot); + Page::Of(object)->RememberCard(slot); } END_LEAF_RUNTIME_ENTRY
diff --git a/runtime/vm/raw_object.h b/runtime/vm/raw_object.h index 625db6d..adcd0c3 100644 --- a/runtime/vm/raw_object.h +++ b/runtime/vm/raw_object.h
@@ -3108,7 +3108,7 @@ friend class ReversePc; template <typename Table, bool kAllCanonicalObjectsAreIncludedIntoSet> friend class CanonicalSetDeserializationCluster; - friend class OldPage; + friend class Page; friend class FastObjectCopy; // For initializing fields. friend void UpdateLengthField(intptr_t, ObjectPtr, ObjectPtr); // length_ };
diff --git a/runtime/vm/service.cc b/runtime/vm/service.cc index 72874dc..27a350e 100644 --- a/runtime/vm/service.cc +++ b/runtime/vm/service.cc
@@ -4592,10 +4592,8 @@ // [anon:dart-*] - as labelled (Android) if ((strcmp(property, "Rss:") == 0) && (size != 0) && (strcmp(path, "(deleted)") != 0) && (strcmp(path, "[heap]") != 0) && - (strcmp(path, "") != 0) && - (strcmp(path, "[anon:dart-newspace]") != 0) && - (strcmp(path, "[anon:dart-oldspace]") != 0) && - (strcmp(path, "[anon:dart-codespace]") != 0) && + (strcmp(path, "") != 0) && (strcmp(path, "[anon:dart-heap]") != 0) && + (strcmp(path, "[anon:dart-code]") != 0) && (strcmp(path, "[anon:dart-profiler]") != 0) && (strcmp(path, "[anon:dart-timeline]") != 0) && (strcmp(path, "[anon:dart-zone]") != 0)) { @@ -4680,9 +4678,9 @@ { JSONObject semi(&vm_children); - semi.AddProperty("name", "SemiSpace Cache"); + semi.AddProperty("name", "Page Cache"); semi.AddProperty("description", "Cached heap regions"); - intptr_t size = SemiSpace::CachedSize(); + intptr_t size = Page::CachedSize(); vm_size += size; semi.AddProperty64("size", size); JSONArray(&semi, "children");
diff --git a/runtime/vm/snapshot.h b/runtime/vm/snapshot.h index 63d61a8..351c52f 100644 --- a/runtime/vm/snapshot.h +++ b/runtime/vm/snapshot.h
@@ -83,7 +83,7 @@ if (!IncludesCode(kind())) { return NULL; } - uword offset = Utils::RoundUp(length(), kMaxObjectAlignment); + uword offset = Utils::RoundUp(length(), kObjectStartAlignment); return Addr() + offset; }
diff --git a/runtime/vm/virtual_memory_compressed.cc b/runtime/vm/virtual_memory_compressed.cc index 7706ea4..7a09940 100644 --- a/runtime/vm/virtual_memory_compressed.cc +++ b/runtime/vm/virtual_memory_compressed.cc
@@ -41,7 +41,7 @@ memset(pages_, 0, kCompressedHeapBitmapSize); ASSERT(size > 0); ASSERT(size <= kCompressedHeapSize); - for (intptr_t page_id = size / kCompressedHeapPageSize; + for (intptr_t page_id = size / kCompressedPageSize; page_id < kCompressedHeapNumPages; page_id++) { SetPageUsed(page_id); } @@ -50,8 +50,8 @@ ASSERT(base_ != 0); ASSERT(size_ != 0); ASSERT(size_ <= kCompressedHeapSize); - ASSERT(Utils::IsAligned(base_, kCompressedHeapPageSize)); - ASSERT(Utils::IsAligned(size_, kCompressedHeapPageSize)); + ASSERT(Utils::IsAligned(base_, kCompressedPageSize)); + ASSERT(Utils::IsAligned(size_, kCompressedPageSize)); // base_ is not necessarily 4GB-aligned, because on some systems we can't make // a large enough reservation to guarantee it. Instead, we have only the // weaker property that all addresses in [base_, base_ + size_) have the same @@ -78,11 +78,10 @@ MemoryRegion VirtualMemoryCompressedHeap::Allocate(intptr_t size, intptr_t alignment) { ASSERT(alignment <= kCompressedHeapAlignment); - const intptr_t allocated_size = Utils::RoundUp(size, kCompressedHeapPageSize); - uword pages = allocated_size / kCompressedHeapPageSize; - uword page_alignment = alignment > kCompressedHeapPageSize - ? alignment / kCompressedHeapPageSize - : 1; + const intptr_t allocated_size = Utils::RoundUp(size, kCompressedPageSize); + uword pages = allocated_size / kCompressedPageSize; + uword page_alignment = + alignment > kCompressedPageSize ? alignment / kCompressedPageSize : 1; MutexLocker ml(mutex_); // Find a gap with enough empty pages, using the bitmap. Note that reading @@ -105,7 +104,7 @@ // Make sure we're not trying to allocate past the end of the heap. uword end = page_id + pages; - if (end > kCompressedHeapSize / kCompressedHeapPageSize) { + if (end > kCompressedHeapSize / kCompressedPageSize) { return MemoryRegion(); } @@ -120,19 +119,19 @@ ++minimum_free_page_id_; } - uword address = base_ + page_id * kCompressedHeapPageSize; - ASSERT(Utils::IsAligned(address, kCompressedHeapPageSize)); + uword address = base_ + page_id * kCompressedPageSize; + ASSERT(Utils::IsAligned(address, kCompressedPageSize)); return MemoryRegion(reinterpret_cast<void*>(address), allocated_size); } void VirtualMemoryCompressedHeap::Free(void* address, intptr_t size) { uword start = reinterpret_cast<uword>(address); - ASSERT(Utils::IsAligned(start, kCompressedHeapPageSize)); - ASSERT(Utils::IsAligned(size, kCompressedHeapPageSize)); + ASSERT(Utils::IsAligned(start, kCompressedPageSize)); + ASSERT(Utils::IsAligned(size, kCompressedPageSize)); MutexLocker ml(mutex_); ASSERT(start >= base_); - uword page_id = (start - base_) / kCompressedHeapPageSize; - uword end = page_id + size / kCompressedHeapPageSize; + uword page_id = (start - base_) / kCompressedPageSize; + uword end = page_id + size / kCompressedPageSize; for (uword i = page_id; i < end; ++i) { ClearPageUsed(i); }
diff --git a/runtime/vm/virtual_memory_compressed.h b/runtime/vm/virtual_memory_compressed.h index bf6fb4c..dbe80c1 100644 --- a/runtime/vm/virtual_memory_compressed.h +++ b/runtime/vm/virtual_memory_compressed.h
@@ -14,9 +14,9 @@ #if defined(DART_COMPRESSED_POINTERS) static constexpr intptr_t kCompressedHeapSize = 4 * GB; static constexpr intptr_t kCompressedHeapAlignment = 4 * GB; -static constexpr intptr_t kCompressedHeapPageSize = kOldPageSize; +static constexpr intptr_t kCompressedPageSize = kPageSize; static constexpr intptr_t kCompressedHeapNumPages = - kCompressedHeapSize / kOldPageSize; + kCompressedHeapSize / kPageSize; static constexpr intptr_t kCompressedHeapBitmapSize = kCompressedHeapNumPages / 8;
diff --git a/runtime/vm/virtual_memory_posix.cc b/runtime/vm/virtual_memory_posix.cc index 72235ff..21f6f79 100644 --- a/runtime/vm/virtual_memory_posix.cc +++ b/runtime/vm/virtual_memory_posix.cc
@@ -133,10 +133,10 @@ // not cross a 4GB boundary. The subregion itself is not necessarily // 4GB-aligned. for (size_t allocated_size = kCompressedHeapSize + kCompressedHeapAlignment; - allocated_size >= kCompressedHeapPageSize; allocated_size >>= 1) { + allocated_size >= kCompressedPageSize; allocated_size >>= 1) { void* address = GenericMapAligned( - nullptr, PROT_NONE, allocated_size, kCompressedHeapPageSize, - allocated_size + kCompressedHeapPageSize, + nullptr, PROT_NONE, allocated_size, kCompressedPageSize, + allocated_size + kCompressedPageSize, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE); if (address == nullptr) continue; @@ -177,7 +177,7 @@ // Also detect for missing support of memfd_create syscall. if (FLAG_dual_map_code) { intptr_t size = PageSize(); - intptr_t alignment = kOldPageSize; + intptr_t alignment = kPageSize; bool executable = true; bool compressed = false; VirtualMemory* vm = @@ -206,7 +206,7 @@ int count = fscanf(fp, "%zu", &max_map_count); fclose(fp); if (count == 1) { - size_t max_heap_pages = FLAG_old_gen_heap_size * MB / kOldPageSize; + size_t max_heap_pages = FLAG_old_gen_heap_size * MB / kPageSize; if (max_map_count < max_heap_pages) { OS::PrintErr( "warning: vm.max_map_count (%zu) is not large enough to support "
diff --git a/runtime/vm/virtual_memory_test.cc b/runtime/vm/virtual_memory_test.cc index c343961..083c906 100644 --- a/runtime/vm/virtual_memory_test.cc +++ b/runtime/vm/virtual_memory_test.cc
@@ -52,7 +52,7 @@ } VM_UNIT_TEST_CASE(AllocateAlignedVirtualMemory) { - intptr_t kHeapPageSize = kOldPageSize; + intptr_t kHeapPageSize = kPageSize; intptr_t kVirtualPageSize = 4096; intptr_t kIterations = kHeapPageSize / kVirtualPageSize;