runtime/vm/runtime_entry.cc

// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

#include "vm/runtime_entry.h"

#include <memory>

#include "platform/address_sanitizer.h"
#include "platform/globals.h"
#include "platform/memory_sanitizer.h"
#include "platform/thread_sanitizer.h"
#include "vm/bootstrap.h"
#include "vm/code_descriptors.h"
#include "vm/code_patcher.h"
#include "vm/compiler/api/deopt_id.h"
#include "vm/compiler/api/type_check_mode.h"
#include "vm/compiler/jit/compiler.h"
#include "vm/dart_api_impl.h"
#include "vm/dart_api_state.h"
#include "vm/dart_entry.h"
#include "vm/debugger.h"
#include "vm/double_conversion.h"
#include "vm/exceptions.h"
#include "vm/ffi_callback_metadata.h"
#include "vm/flags.h"
#include "vm/heap/verifier.h"
#include "vm/instructions.h"
#include "vm/interpreter.h"
#include "vm/kernel_isolate.h"
#include "vm/log.h"
#include "vm/message.h"
#include "vm/message_handler.h"
#include "vm/object_store.h"
#include "vm/parser.h"
#include "vm/resolver.h"
#include "vm/service_isolate.h"
#include "vm/stack_frame.h"
#include "vm/symbols.h"
#include "vm/thread.h"
#include "vm/type_testing_stubs.h"
#include "vm/zone_text_buffer.h"

#if !defined(DART_PRECOMPILED_RUNTIME)
#include "vm/compiler/backend/locations.h"
#include "vm/compiler/ffi/marshaller.h"
#include "vm/deopt_instructions.h"
#endif  // !defined(DART_PRECOMPILED_RUNTIME)

namespace dart {

static constexpr intptr_t kDefaultMaxSubtypeCacheEntries =
    SubtypeTestCache::MaxEntriesForCacheAllocatedFor(1000);
DEFINE_FLAG(
    int,
    max_subtype_cache_entries,
    kDefaultMaxSubtypeCacheEntries,
    "Maximum number of subtype cache entries (number of checks cached).");
DEFINE_FLAG(int,
            reoptimization_counter_threshold,
            4000,
            "Counter threshold before a function gets reoptimized.");
DEFINE_FLAG(bool,
            runtime_allocate_old,
            false,
            "Use old-space for allocation via runtime calls.");
DEFINE_FLAG(bool,
            runtime_allocate_spill_tlab,
            false,
            "Ensure results of allocation via runtime calls are not in an "
            "active TLAB.");
DEFINE_FLAG(bool, trace_deoptimization, false, "Trace deoptimization");
DEFINE_FLAG(bool,
            trace_deoptimization_verbose,
            false,
            "Trace deoptimization verbose");

DECLARE_FLAG(int, max_deoptimization_counter_threshold);
DECLARE_FLAG(bool, trace_compiler);
DECLARE_FLAG(bool, trace_optimizing_compiler);
DECLARE_FLAG(int, max_polymorphic_checks);

DEFINE_FLAG(bool, trace_osr, false, "Trace attempts at on-stack replacement.");

DEFINE_FLAG(int, gc_every, 0, "Run major GC on every N stack overflow checks");
DEFINE_FLAG(int,
            stacktrace_every,
            0,
            "Compute debugger stacktrace on every N stack overflow checks");
DEFINE_FLAG(charp,
            stacktrace_filter,
            nullptr,
            "Compute stacktrace in named function on stack overflow checks");
DEFINE_FLAG(charp,
            deoptimize_filter,
            nullptr,
            "Deoptimize in named function on stack overflow checks");
DEFINE_FLAG(charp,
            deoptimize_on_runtime_call_name_filter,
            nullptr,
            "Runtime call name filter for --deoptimize-on-runtime-call-every.");

DEFINE_FLAG(bool,
            unopt_monomorphic_calls,
            true,
            "Enable specializing monomorphic calls from unoptimized code.");
DEFINE_FLAG(bool,
            unopt_megamorphic_calls,
            true,
            "Enable specializing megamorphic calls from unoptimized code.");
DEFINE_FLAG(bool,
            verbose_stack_overflow,
            false,
            "Print additional details about stack overflow.");
DEFINE_FLAG(bool, gc_at_throw, false, "Run evacuating GC at throw and rethrow");

DECLARE_FLAG(int, reload_every);
DECLARE_FLAG(bool, reload_every_optimized);
DECLARE_FLAG(bool, reload_every_back_off);

uword RuntimeEntry::GetEntryPoint() const {
  // Compute the effective address. When running under the simulator,
  // this is a redirection address that forces the simulator to call
  // into the runtime system.
  uword entry = reinterpret_cast<uword>(function());
#if defined(DART_INCLUDE_SIMULATOR)
  if (FLAG_use_simulator) {
    // Redirection to leaf runtime calls supports a maximum of 4 arguments
    // passed in registers (maximum 2 double arguments for leaf float runtime
    // calls).
    ASSERT(argument_count() >= 0);
    ASSERT(!is_leaf() || (!is_float() && (argument_count() <= 4)) ||
           (argument_count() <= 2));
    Simulator::CallKind call_kind =
        is_leaf() ? (is_float() ? Simulator::kLeafFloatRuntimeCall
                                : Simulator::kLeafRuntimeCall)
                  : Simulator::kRuntimeCall;
    entry = Simulator::RedirectExternalReference(entry, call_kind,
                                                 argument_count());
  }
#endif
  return entry;
}

#ifdef DEBUG
#define TRACE_RUNTIME_CALL(format, name)                                       \
  if (FLAG_trace_runtime_calls) {                                              \
    THR_Print("Runtime call: " format "\n", name);                             \
  }
#else
#define TRACE_RUNTIME_CALL(format, name)                                       \
  do {                                                                         \
  } while (0)
#endif

#if defined(DART_INCLUDE_SIMULATOR)
#define CHECK_SIMULATOR_STACK_OVERFLOW()                                       \
  if (FLAG_use_simulator && !OSThread::Current()->HasStackHeadroom()) {        \
    Exceptions::ThrowStackOverflow();                                          \
  }
#else
#define CHECK_SIMULATOR_STACK_OVERFLOW()
#endif  // defined(DART_INCLUDE_SIMULATOR)

void OnEveryRuntimeEntryCall(Thread* thread,
                             const char* runtime_call_name,
                             bool can_lazy_deopt);

#define DEFINE_RUNTIME_ENTRY_IMPL(name, argument_count, can_lazy_deopt)        \
  extern void DRT_##name(NativeArguments arguments);                           \
  extern const RuntimeEntry k##name##RuntimeEntry(                             \
      "DRT_" #name, reinterpret_cast<const void*>(DRT_##name), argument_count, \
      false, false, can_lazy_deopt);                                           \
  static void DRT_Helper##name(Isolate* isolate, Thread* thread, Zone* zone,   \
                               NativeArguments arguments);                     \
  extern "C" void DRT_##name(NativeArguments arguments) {                      \
    CHECK_STACK_ALIGNMENT;                                                     \
    /* Tell MemorySanitizer 'arguments' is initialized by generated code. */   \
    MSAN_UNPOISON(&arguments, sizeof(arguments));                              \
    ASSERT(arguments.ArgCount() == argument_count);                            \
    TRACE_RUNTIME_CALL("%s", "" #name);                                        \
    {                                                                          \
      Thread* thread = arguments.thread();                                     \
      ASSERT(thread == Thread::Current());                                     \
      RuntimeCallDeoptScope runtime_call_deopt_scope(                          \
          thread, can_lazy_deopt ? RuntimeCallDeoptAbility::kCanLazyDeopt      \
                                 : RuntimeCallDeoptAbility::kCannotLazyDeopt); \
      Isolate* isolate = thread->isolate();                                    \
      TransitionGeneratedToVM transition(thread);                              \
      StackZone zone(thread);                                                  \
      CHECK_SIMULATOR_STACK_OVERFLOW();                                        \
      if (FLAG_deoptimize_on_runtime_call_every > 0) {                         \
        OnEveryRuntimeEntryCall(thread, "" #name, can_lazy_deopt);             \
      }                                                                        \
      DRT_Helper##name(isolate, thread, zone.GetZone(), arguments);            \
    }                                                                          \
  }                                                                            \
  static void DRT_Helper##name(Isolate* isolate, Thread* thread, Zone* zone,   \
                               NativeArguments arguments)

#define DEFINE_RUNTIME_ENTRY(name, argument_count)                             \
  DEFINE_RUNTIME_ENTRY_IMPL(name, argument_count, /*can_lazy_deopt=*/true)

#define DEFINE_RUNTIME_ENTRY_NO_LAZY_DEOPT(name, argument_count)               \
  DEFINE_RUNTIME_ENTRY_IMPL(name, argument_count, /*can_lazy_deopt=*/false)

#define DEFINE_LEAF_RUNTIME_ENTRY(name, argument_count, func)                  \
  extern const RuntimeEntry k##name##RuntimeEntry(                             \
      "DLRT_" #name, reinterpret_cast<const void*>(func), argument_count,      \
      true, false, /*can_lazy_deopt=*/false)

#define DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(name, argument_count, func)            \
  extern const RuntimeEntry k##name##RuntimeEntry(                             \
      "DLRT_" #name, reinterpret_cast<const void*>(func), argument_count,      \
      true, true, /*can_lazy_deopt=*/false)

// Helper returning the token position of the Dart caller.
static TokenPosition GetCallerLocation() {
  DartFrameIterator iterator(Thread::Current(),
                             StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* caller_frame = iterator.NextFrame();
  ASSERT(caller_frame != nullptr);
  return caller_frame->GetTokenPos();
}

DEFINE_RUNTIME_ENTRY(RangeError, 2) {
  const Instance& length = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const Instance& index = Instance::CheckedHandle(zone, arguments.ArgAt(1));
  if (!length.IsInteger()) {
    // Throw: new ArgumentError.value(length, "length", "is not an integer");
    const Array& args = Array::Handle(zone, Array::New(3));
    args.SetAt(0, length);
    args.SetAt(1, Symbols::Length());
    args.SetAt(2, String::Handle(zone, String::New("is not an integer")));
    Exceptions::ThrowByType(Exceptions::kArgumentValue, args);
  }
  if (!index.IsInteger()) {
    // Throw: new ArgumentError.value(index, "index", "is not an integer");
    const Array& args = Array::Handle(zone, Array::New(3));
    args.SetAt(0, index);
    args.SetAt(1, Symbols::Index());
    args.SetAt(2, String::Handle(zone, String::New("is not an integer")));
    Exceptions::ThrowByType(Exceptions::kArgumentValue, args);
  }
  // Throw: new RangeError.range(index, 0, length - 1, "length");
  const Array& args = Array::Handle(zone, Array::New(4));
  args.SetAt(0, index);
  args.SetAt(1, Integer::Handle(zone, Integer::New(0)));
  args.SetAt(
      2, Integer::Handle(
             zone, Integer::Cast(length).ArithmeticOp(
                       Token::kSUB, Integer::Handle(zone, Integer::New(1)))));
  args.SetAt(3, Symbols::Length());
  Exceptions::ThrowByType(Exceptions::kRange, args);
}

DEFINE_RUNTIME_ENTRY(RangeErrorUnboxedInt64, 0) {
  int64_t unboxed_length = thread->unboxed_int64_runtime_arg();
  int64_t unboxed_index = thread->unboxed_int64_runtime_second_arg();
  const auto& length = Integer::Handle(zone, Integer::New(unboxed_length));
  const auto& index = Integer::Handle(zone, Integer::New(unboxed_index));
  // Throw: new RangeError.range(index, 0, length - 1, "length");
  const Array& args = Array::Handle(zone, Array::New(4));
  args.SetAt(0, index);
  args.SetAt(1, Integer::Handle(zone, Integer::New(0)));
  args.SetAt(
      2, Integer::Handle(
             zone, Integer::Cast(length).ArithmeticOp(
                       Token::kSUB, Integer::Handle(zone, Integer::New(1)))));
  args.SetAt(3, Symbols::Length());
  Exceptions::ThrowByType(Exceptions::kRange, args);
}

DEFINE_RUNTIME_ENTRY(WriteError, 2) {
  const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const Smi& kind = Smi::CheckedHandle(zone, arguments.ArgAt(1));
  auto& message = String::Handle(zone);
  switch (kind.Value()) {
    case 0:  // CheckWritableInstr::Kind::kWriteUnmodifiableTypedData:
      message = String::NewFormatted("Cannot modify an unmodifiable list: %s",
                                     receiver.ToCString());
      break;
    case 1:  // CheckWritableInstr::Kind::kDeeplyImmutableAttachNativeFinalizer:
      message = String::NewFormatted(
          "Cannot attach NativeFinalizer to deeply immutable object: %s",
          receiver.ToCString());
      break;
  }
  const Array& args = Array::Handle(Array::New(1));
  args.SetAt(0, message);
  Exceptions::ThrowByType(Exceptions::kUnsupported, args);
}

static void NullErrorHelper(Zone* zone,
                            const String& selector,
                            bool is_param_name = false) {
  if (is_param_name) {
    const String& error = String::Handle(
        selector.IsNull()
            ? String::New("argument value is null")
            : String::NewFormatted("argument value for '%s' is null",
                                   selector.ToCString()));
    Exceptions::ThrowArgumentError(error);
    return;
  }

  // If the selector is null, this must be a null check that wasn't due to a
  // method invocation, so was due to the null check operator.
  if (selector.IsNull()) {
    const Array& args = Array::Handle(zone, Array::New(4));
    args.SetAt(
        3, String::Handle(
               zone, String::New("Null check operator used on a null value")));
    Exceptions::ThrowByType(Exceptions::kType, args);
    return;
  }

  InvocationMirror::Kind kind = InvocationMirror::kMethod;
  if (Field::IsGetterName(selector)) {
    kind = InvocationMirror::kGetter;
  } else if (Field::IsSetterName(selector)) {
    kind = InvocationMirror::kSetter;
  }

  const Smi& invocation_type = Smi::Handle(
      zone,
      Smi::New(InvocationMirror::EncodeType(InvocationMirror::kDynamic, kind)));

  const Array& args = Array::Handle(zone, Array::New(7));
  args.SetAt(0, /* instance */ Object::null_object());
  args.SetAt(1, selector);
  args.SetAt(2, invocation_type);
  args.SetAt(3, /* func_type_args_length */ Object::smi_zero());
  args.SetAt(4, /* func_type_args */ Object::null_object());
  args.SetAt(5, /* func_args */ Object::null_object());
  args.SetAt(6, /* func_arg_names */ Object::null_object());
  Exceptions::ThrowByType(Exceptions::kNoSuchMethod, args);
}

static void DoThrowNullError(Thread* thread, Zone* zone, bool is_param) {
  DartFrameIterator iterator(thread,
                             StackFrameIterator::kNoCrossThreadIteration);
  const StackFrame* caller_frame = iterator.NextFrame();
  ASSERT(caller_frame->IsDartFrame());
  ASSERT(!caller_frame->is_interpreted());
  const Code& code = Code::Handle(zone, caller_frame->LookupDartCode());
  const uword pc_offset = caller_frame->pc() - code.PayloadStart();

  if (FLAG_shared_slow_path_triggers_gc) {
    thread->isolate_group()->heap()->CollectAllGarbage(GCReason::kDebugging);
  }

  const CodeSourceMap& map =
      CodeSourceMap::Handle(zone, code.code_source_map());
  String& member_name = String::Handle(zone);
  if (!map.IsNull()) {
    CodeSourceMapReader reader(map, Array::null_array(),
                               Function::null_function());
    const intptr_t name_index = reader.GetNullCheckNameIndexAt(pc_offset);
    RELEASE_ASSERT(name_index >= 0);

    const ObjectPool& pool = ObjectPool::Handle(zone, code.GetObjectPool());
    member_name ^= pool.ObjectAt(name_index);
  } else {
    member_name = Symbols::OptimizedOut().ptr();
  }

  NullErrorHelper(zone, member_name, is_param);
}

DEFINE_RUNTIME_ENTRY(NullError, 0) {
  DoThrowNullError(thread, zone, /*is_param=*/false);
}

// Collects information about pointers within the top |kMaxSlotsCollected|
// slots on the stack.
// TODO(b/179632636) This code is added in attempt to better understand
// b/179632636 and should be removed in the future.
void ReportImpossibleNullError(intptr_t cid,
                               StackFrame* caller_frame,
                               Thread* thread) {
  TextBuffer buffer(512);
  buffer.Printf("hit null error with cid %" Pd ", caller context: ", cid);

  const intptr_t kMaxSlotsCollected = 5;
  const auto slots = reinterpret_cast<ObjectPtr*>(caller_frame->sp());
  const intptr_t num_slots_in_frame =
      reinterpret_cast<ObjectPtr*>(caller_frame->fp()) - slots;
  const auto num_slots_to_collect =
      Utils::Maximum(kMaxSlotsCollected, num_slots_in_frame);
  bool comma = false;
  for (intptr_t i = 0; i < num_slots_to_collect; i++) {
    const ObjectPtr ptr = slots[i];
    buffer.Printf("%s[sp+%" Pd "] %" Pp "", comma ? ", " : "", i,
                  static_cast<uword>(ptr));
    if (ptr->IsHeapObject() &&
        thread->heap()->Contains(UntaggedObject::ToAddr(ptr))) {
      buffer.Printf("(%" Pp ")", static_cast<uword>(ptr->untag()->tags_));
    }
    comma = true;
  }

  const char* message = buffer.buffer();
  FATAL("%s", message);
}

DEFINE_RUNTIME_ENTRY(DispatchTableNullError, 1) {
  const Smi& cid = Smi::CheckedHandle(zone, arguments.ArgAt(0));
  if (cid.Value() != kNullCid) {
    // We hit null error, but receiver is not null itself. This most likely
    // is a memory corruption. Crash the VM but provide some additional
    // information about the arguments on the stack.
    DartFrameIterator iterator(thread,
                               StackFrameIterator::kNoCrossThreadIteration);
    StackFrame* caller_frame = iterator.NextFrame();
    RELEASE_ASSERT(caller_frame->IsDartFrame());
    ReportImpossibleNullError(cid.Value(), caller_frame, thread);
  }
  DoThrowNullError(thread, zone, /*is_param=*/false);
}

DEFINE_RUNTIME_ENTRY(NullErrorWithSelector, 1) {
  const String& selector = String::CheckedHandle(zone, arguments.ArgAt(0));
  NullErrorHelper(zone, selector);
}

DEFINE_RUNTIME_ENTRY(NullCastError, 0) {
  NullErrorHelper(zone, String::null_string());
}

DEFINE_RUNTIME_ENTRY(TypeError, 2) {
  const Instance& src_instance =
      Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const AbstractType& dst_type =
      AbstractType::CheckedHandle(zone, arguments.ArgAt(1));
  const TokenPosition location = GetCallerLocation();
  const auto& src_type =
      AbstractType::Handle(zone, src_instance.GetType(Heap::kNew));
  Exceptions::CreateAndThrowTypeError(location, src_type, dst_type,
                                      Symbols::Empty());
}

DEFINE_RUNTIME_ENTRY(ArgumentNullError, 0) {
  DoThrowNullError(thread, zone, /*is_param=*/true);
}

DEFINE_RUNTIME_ENTRY(ArgumentError, 1) {
  const Instance& value = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  Exceptions::ThrowArgumentError(value);
}

DEFINE_RUNTIME_ENTRY(ArgumentErrorUnboxedInt64, 0) {
  // Unboxed value is passed through a dedicated slot in Thread.
  int64_t unboxed_value = arguments.thread()->unboxed_int64_runtime_arg();
  const Integer& value = Integer::Handle(zone, Integer::New(unboxed_value));
  Exceptions::ThrowArgumentError(value);
}

DEFINE_RUNTIME_ENTRY(UnsupportedError, 1) {
  const auto& message = String::CheckedHandle(zone, arguments.ArgAt(0));
  const Array& args = Array::Handle(Array::New(1));
  args.SetAt(0, message);
  Exceptions::ThrowByType(Exceptions::kUnsupported, args);
}

DEFINE_RUNTIME_ENTRY(DoubleToInteger, 1) {
  // Unboxed value is passed through a dedicated slot in Thread.
  double val = arguments.thread()->unboxed_double_runtime_arg();
  const Smi& recognized_kind = Smi::CheckedHandle(zone, arguments.ArgAt(0));
  switch (recognized_kind.Value()) {
    case MethodRecognizer::kDoubleToInteger:
      break;
    case MethodRecognizer::kDoubleFloorToInt:
      val = floor(val);
      break;
    case MethodRecognizer::kDoubleCeilToInt:
      val = ceil(val);
      break;
    default:
      UNREACHABLE();
  }
  arguments.SetReturn(Integer::Handle(zone, DoubleToInteger(zone, val)));
}

DEFINE_RUNTIME_ENTRY(IntegerDivisionByZeroException, 0) {
  const Array& args = Array::Handle(zone, Array::New(0));
  Exceptions::ThrowByType(Exceptions::kIntegerDivisionByZeroException, args);
}

static Heap::Space SpaceForRuntimeAllocation() {
  if (FLAG_runtime_allocate_old) [[unlikely]] {
    return Heap::kOld;
  } else {
    return Heap::kNew;
  }
}

static void RuntimeAllocationEpilogue(Thread* thread) {
  if (FLAG_runtime_allocate_spill_tlab) [[unlikely]] {
    static RelaxedAtomic<uword> count = 0;
    if ((count++ % 10) == 0) {
      thread->heap()->new_space()->AbandonRemainingTLAB(thread);
    }
  }
}

// Allocation of a fixed length array of given element type.
// This runtime entry is never called for allocating a List of a generic type,
// because a prior run time call instantiates the element type if necessary.
// Arg0: array length.
// Arg1: array type arguments, i.e. vector of 1 type, the element type.
// Return value: newly allocated array of length arg0.
DEFINE_RUNTIME_ENTRY(AllocateArray, 2) {
  const Instance& length = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  if (!length.IsInteger()) {
    // Throw: new ArgumentError.value(length, "length", "is not an integer");
    const Array& args = Array::Handle(zone, Array::New(3));
    args.SetAt(0, length);
    args.SetAt(1, Symbols::Length());
    args.SetAt(2, String::Handle(zone, String::New("is not an integer")));
    Exceptions::ThrowByType(Exceptions::kArgumentValue, args);
  }
  const int64_t len = Integer::Cast(length).Value();
  if (len < 0) {
    // Throw: new RangeError.range(length, 0, Array::kMaxElements, "length");
    Exceptions::ThrowRangeError("length", Integer::Cast(length), 0,
                                Array::kMaxElements);
  }
  if (len > Array::kMaxElements) {
    Exceptions::ThrowOOM();
  }

  const Array& array = Array::Handle(
      zone,
      Array::New(static_cast<intptr_t>(len), SpaceForRuntimeAllocation()));
  TypeArguments& element_type =
      TypeArguments::CheckedHandle(zone, arguments.ArgAt(1));
  // An Array is raw or takes one type argument. However, its type argument
  // vector may be longer than 1 due to a type optimization reusing the type
  // argument vector of the instantiator.
  ASSERT(element_type.IsNull() ||
         (element_type.Length() >= 1 && element_type.IsInstantiated()));
  array.SetTypeArguments(element_type);  // May be null.
  arguments.SetReturn(array);
  RuntimeAllocationEpilogue(thread);
}

DEFINE_RUNTIME_ENTRY_NO_LAZY_DEOPT(AllocateMint, 0) {
  if (FLAG_shared_slow_path_triggers_gc) {
    thread->isolate_group()->heap()->CollectAllGarbage(GCReason::kDebugging);
  }
  arguments.SetReturn(Object::Handle(
      zone, Integer::New(kMaxInt64, SpaceForRuntimeAllocation())));
  RuntimeAllocationEpilogue(thread);
}

DEFINE_RUNTIME_ENTRY_NO_LAZY_DEOPT(BoxInt, 0) {
  const int64_t val = thread->unboxed_int64_runtime_arg();
  arguments.SetReturn(
      Object::Handle(zone, Integer::New(val, SpaceForRuntimeAllocation())));
  RuntimeAllocationEpilogue(thread);
}

DEFINE_RUNTIME_ENTRY_NO_LAZY_DEOPT(BoxDouble, 0) {
  const double val = thread->unboxed_double_runtime_arg();
  arguments.SetReturn(
      Object::Handle(zone, Double::New(val, SpaceForRuntimeAllocation())));
  RuntimeAllocationEpilogue(thread);
}

DEFINE_RUNTIME_ENTRY_NO_LAZY_DEOPT(BoxFloat32x4, 0) {
  const auto val = thread->unboxed_simd128_runtime_arg();
  arguments.SetReturn(
      Object::Handle(zone, Float32x4::New(val, SpaceForRuntimeAllocation())));
  RuntimeAllocationEpilogue(thread);
}

DEFINE_RUNTIME_ENTRY_NO_LAZY_DEOPT(BoxFloat64x2, 0) {
  const auto val = thread->unboxed_simd128_runtime_arg();
  arguments.SetReturn(
      Object::Handle(zone, Float64x2::New(val, SpaceForRuntimeAllocation())));
  RuntimeAllocationEpilogue(thread);
}

DEFINE_RUNTIME_ENTRY_NO_LAZY_DEOPT(BoxInt32x4, 0) {
  const auto val = thread->unboxed_simd128_runtime_arg();
  arguments.SetReturn(
      Object::Handle(zone, Int32x4::New(val, SpaceForRuntimeAllocation())));
  RuntimeAllocationEpilogue(thread);
}

DEFINE_RUNTIME_ENTRY_NO_LAZY_DEOPT(AllocateDouble, 0) {
  if (FLAG_shared_slow_path_triggers_gc) {
    thread->isolate_group()->heap()->CollectAllGarbage(GCReason::kDebugging);
  }
  arguments.SetReturn(
      Object::Handle(zone, Double::New(0.0, SpaceForRuntimeAllocation())));
  RuntimeAllocationEpilogue(thread);
}

DEFINE_RUNTIME_ENTRY_NO_LAZY_DEOPT(AllocateFloat32x4, 0) {
  if (FLAG_shared_slow_path_triggers_gc) {
    thread->isolate_group()->heap()->CollectAllGarbage(GCReason::kDebugging);
  }
  arguments.SetReturn(Object::Handle(
      zone, Float32x4::New(0.0, 0.0, 0.0, 0.0, SpaceForRuntimeAllocation())));
  RuntimeAllocationEpilogue(thread);
}

DEFINE_RUNTIME_ENTRY_NO_LAZY_DEOPT(AllocateFloat64x2, 0) {
  if (FLAG_shared_slow_path_triggers_gc) {
    thread->isolate_group()->heap()->CollectAllGarbage(GCReason::kDebugging);
  }
  arguments.SetReturn(Object::Handle(
      zone, Float64x2::New(0.0, 0.0, SpaceForRuntimeAllocation())));
  RuntimeAllocationEpilogue(thread);
}

DEFINE_RUNTIME_ENTRY_NO_LAZY_DEOPT(AllocateInt32x4, 0) {
  if (FLAG_shared_slow_path_triggers_gc) {
    thread->isolate_group()->heap()->CollectAllGarbage(GCReason::kDebugging);
  }
  arguments.SetReturn(Object::Handle(
      zone, Int32x4::New(0, 0, 0, 0, SpaceForRuntimeAllocation())));
  RuntimeAllocationEpilogue(thread);
}

// Allocate typed data array of given class id and length.
// Arg0: class id.
// Arg1: number of elements.
// Return value: newly allocated typed data array.
DEFINE_RUNTIME_ENTRY(AllocateTypedData, 2) {
  const intptr_t cid = Smi::CheckedHandle(zone, arguments.ArgAt(0)).Value();
  const auto& length = Instance::CheckedHandle(zone, arguments.ArgAt(1));
  if (!length.IsInteger()) {
    const Array& args = Array::Handle(zone, Array::New(1));
    args.SetAt(0, length);
    Exceptions::ThrowByType(Exceptions::kArgument, args);
  }
  const int64_t len = Integer::Cast(length).Value();
  const intptr_t max = TypedData::MaxElements(cid);
  if (len < 0) {
    Exceptions::ThrowRangeError("length", Integer::Cast(length), 0, max);
  } else if (len > max) {
    Exceptions::ThrowOOM();
  }
  const auto& typed_data =
      TypedData::Handle(zone, TypedData::New(cid, static_cast<intptr_t>(len),
                                             SpaceForRuntimeAllocation()));
  arguments.SetReturn(typed_data);
  RuntimeAllocationEpilogue(thread);
}

// Allocate OneByteString of given length.
// Arg0: number of elements.
// Return value: newly allocated OneByteString.
DEFINE_RUNTIME_ENTRY(AllocateOneByteString, 1) {
  const int64_t length =
      Integer::CheckedHandle(zone, arguments.ArgAt(0)).Value();
  if ((length < 0) || (length > OneByteString::kMaxElements)) {
    // Assume that negative lengths are the result of wrapping in code in
    // string_patch.dart.
    Exceptions::ThrowOOM();
  }
  const auto& str =
      String::Handle(zone, OneByteString::New(static_cast<intptr_t>(length),
                                              SpaceForRuntimeAllocation()));
  arguments.SetReturn(str);
  RuntimeAllocationEpilogue(thread);
}

// Allocate TwoByteString of given length.
// Arg0: number of elements.
// Return value: newly allocated TwoByteString.
DEFINE_RUNTIME_ENTRY(AllocateTwoByteString, 1) {
  const int64_t length =
      Integer::CheckedHandle(zone, arguments.ArgAt(0)).Value();
  if ((length < 0) || (length > TwoByteString::kMaxElements)) {
    // Assume that negative lengths are the result of wrapping in code in
    // string_patch.dart.
    Exceptions::ThrowOOM();
  }
  const auto& str =
      String::Handle(zone, TwoByteString::New(static_cast<intptr_t>(length),
                                              SpaceForRuntimeAllocation()));
  arguments.SetReturn(str);
  RuntimeAllocationEpilogue(thread);
}

// Result of an invoke may be an unhandled exception, in which case we
// rethrow it.
static void ThrowIfError(const Object& result) {
  if (!result.IsNull() && result.IsError()) {
    Exceptions::PropagateError(Error::Cast(result));
  }
}

// Allocate a new object.
// Arg0: class of the object that needs to be allocated.
// Arg1: type arguments of the object that needs to be allocated.
// Return value: newly allocated object.
DEFINE_RUNTIME_ENTRY(AllocateObject, 2) {
  const Class& cls = Class::CheckedHandle(zone, arguments.ArgAt(0));
#if defined(DART_DYNAMIC_MODULES) && !defined(DART_PRECOMPILED_RUNTIME)
  if (!cls.is_allocate_finalized()) {
    const Error& error =
        Error::Handle(zone, cls.EnsureIsAllocateFinalized(thread));
    if (!error.IsNull()) {
      Exceptions::PropagateError(error);
      UNREACHABLE();
    }
  }
#endif
  ASSERT(cls.is_allocate_finalized());
  const Instance& instance = Instance::Handle(
      zone, Instance::NewAlreadyFinalized(cls, SpaceForRuntimeAllocation()));
  if (cls.NumTypeArguments() == 0) {
    // No type arguments required for a non-parameterized type.
    ASSERT(Instance::CheckedHandle(zone, arguments.ArgAt(1)).IsNull());
  } else {
    const auto& type_arguments =
        TypeArguments::CheckedHandle(zone, arguments.ArgAt(1));
    // Unless null (for a raw type), the type argument vector may be longer than
    // necessary due to a type optimization reusing the type argument vector of
    // the instantiator.
    ASSERT(type_arguments.IsNull() ||
           (type_arguments.IsInstantiated() &&
            (type_arguments.Length() >= cls.NumTypeArguments())));
    instance.SetTypeArguments(type_arguments);
  }
  arguments.SetReturn(instance);
  RuntimeAllocationEpilogue(thread);
}

extern "C" uword /*ObjectPtr*/ DLRT_EnsureRememberedAndMarkingDeferred(
    uword /*ObjectPtr*/ object_in,
    Thread* thread) {
  ObjectPtr object = static_cast<ObjectPtr>(object_in);

  // If we eliminate the generational write barrier when writing into an object,
  // we need to ensure it's either a new-space object or it has been added to
  // the remembered set. If we eliminate the incremental write barrier, we need
  // to add the object to the deferred marking stack so it will be [re]scanned.
  //
  // NOTE: We use static_cast<>() instead of ::RawCast() to avoid handle
  // allocations in debug mode. Handle allocations in leaf runtimes can cause
  // memory leaks because they will allocate into a handle scope from the next
  // outermost runtime code (to which the generated Dart code might not return
  // in a long time).
  bool skips_barrier = true;
  if (object->IsArray()) {
    const intptr_t length = Array::LengthOf(static_cast<ArrayPtr>(object));
    skips_barrier = compiler::target::WillAllocateNewOrRememberedArray(length);
  } else if (object->IsContext()) {
    const intptr_t num_context_variables =
        Context::NumVariables(static_cast<ContextPtr>(object));
    skips_barrier = compiler::target::WillAllocateNewOrRememberedContext(
        num_context_variables);
  }

  if (skips_barrier) {
    if (object->IsOldObject()) {
      object->untag()->EnsureInRememberedSet(thread);
    }

    if (thread->is_marking()) {
      thread->DeferredMarkingStackAddObject(object);
    }
  }

  return static_cast<uword>(object);
}
DEFINE_LEAF_RUNTIME_ENTRY(EnsureRememberedAndMarkingDeferred,
                          2,
                          DLRT_EnsureRememberedAndMarkingDeferred);

extern "C" void DLRT_StoreBufferBlockProcess(Thread* thread) {
  thread->StoreBufferBlockProcess(StoreBuffer::kCheckThreshold);
}
DEFINE_LEAF_RUNTIME_ENTRY(StoreBufferBlockProcess,
                          1,
                          DLRT_StoreBufferBlockProcess);

extern "C" void DLRT_OldMarkingStackBlockProcess(Thread* thread) {
  thread->OldMarkingStackBlockProcess();
}
DEFINE_LEAF_RUNTIME_ENTRY(OldMarkingStackBlockProcess,
                          1,
                          DLRT_OldMarkingStackBlockProcess);

extern "C" void DLRT_NewMarkingStackBlockProcess(Thread* thread) {
  thread->NewMarkingStackBlockProcess();
}
DEFINE_LEAF_RUNTIME_ENTRY(NewMarkingStackBlockProcess,
                          1,
                          DLRT_NewMarkingStackBlockProcess);

// Instantiate type.
// Arg0: uninstantiated type.
// Arg1: instantiator type arguments.
// Arg2: function type arguments.
// Return value: instantiated type.
DEFINE_RUNTIME_ENTRY(InstantiateType, 3) {
  AbstractType& type = AbstractType::CheckedHandle(zone, arguments.ArgAt(0));
  const TypeArguments& instantiator_type_arguments =
      TypeArguments::CheckedHandle(zone, arguments.ArgAt(1));
  const TypeArguments& function_type_arguments =
      TypeArguments::CheckedHandle(zone, arguments.ArgAt(2));
  ASSERT(!type.IsNull());
  ASSERT(instantiator_type_arguments.IsNull() ||
         instantiator_type_arguments.IsInstantiated());
  ASSERT(function_type_arguments.IsNull() ||
         function_type_arguments.IsInstantiated());
  type = type.InstantiateFrom(instantiator_type_arguments,
                              function_type_arguments, kAllFree, Heap::kOld);
  ASSERT(!type.IsNull() && type.IsInstantiated());
  arguments.SetReturn(type);
}

// Instantiate type arguments.
// Arg0: uninstantiated type arguments.
// Arg1: instantiator type arguments.
// Arg2: function type arguments.
// Return value: instantiated type arguments.
DEFINE_RUNTIME_ENTRY(InstantiateTypeArguments, 3) {
  TypeArguments& type_arguments =
      TypeArguments::CheckedHandle(zone, arguments.ArgAt(0));
  const TypeArguments& instantiator_type_arguments =
      TypeArguments::CheckedHandle(zone, arguments.ArgAt(1));
  const TypeArguments& function_type_arguments =
      TypeArguments::CheckedHandle(zone, arguments.ArgAt(2));
  ASSERT(!type_arguments.IsNull() && !type_arguments.IsInstantiated());
  ASSERT(instantiator_type_arguments.IsNull() ||
         instantiator_type_arguments.IsInstantiated());
  ASSERT(function_type_arguments.IsNull() ||
         function_type_arguments.IsInstantiated());
  // Code inlined in the caller should have optimized the case where the
  // instantiator can be reused as type argument vector.
  ASSERT(!type_arguments.IsUninstantiatedIdentity());
  type_arguments = type_arguments.InstantiateAndCanonicalizeFrom(
      instantiator_type_arguments, function_type_arguments);
  ASSERT(type_arguments.IsNull() || type_arguments.IsInstantiated());
  arguments.SetReturn(type_arguments);
}

// Helper routine for tracing a subtype check.
static void PrintSubtypeCheck(const AbstractType& subtype,
                              const AbstractType& supertype,
                              const bool result) {
  DartFrameIterator iterator(Thread::Current(),
                             StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* caller_frame = iterator.NextFrame();
  ASSERT(caller_frame != nullptr);

  LogBlock lb;
  THR_Print("SubtypeCheck: '%s' %d %s '%s' %d (pc: %#" Px ").\n",
            subtype.NameCString(), subtype.type_class_id(),
            result ? "is" : "is !", supertype.NameCString(),
            supertype.type_class_id(), caller_frame->pc());

  const Function& function =
      Function::Handle(caller_frame->LookupDartFunction());
  if (function.HasSavedArgumentsDescriptor()) {
    const auto& args_desc_array = Array::Handle(function.saved_args_desc());
    const ArgumentsDescriptor args_desc(args_desc_array);
    THR_Print(" -> Function %s [%s]\n", function.ToFullyQualifiedCString(),
              args_desc.ToCString());
  } else {
    THR_Print(" -> Function %s\n", function.ToFullyQualifiedCString());
  }
}

// Assert supertype-subtype relationship.
// Arg0: instantiator type arguments
// Arg1: function type arguments
// Arg2: type to be a subtype of the other
// Arg3: type to be a supertype of the other
// Arg4: variable name of the subtype parameter
// No return value.
DEFINE_RUNTIME_ENTRY(SubtypeCheck, 5) {
  const TypeArguments& instantiator_type_args =
      TypeArguments::CheckedHandle(zone, arguments.ArgAt(0));
  const TypeArguments& function_type_args =
      TypeArguments::CheckedHandle(zone, arguments.ArgAt(1));
  AbstractType& subtype = AbstractType::CheckedHandle(zone, arguments.ArgAt(2));
  AbstractType& supertype =
      AbstractType::CheckedHandle(zone, arguments.ArgAt(3));
  const String& dst_name = String::CheckedHandle(zone, arguments.ArgAt(4));

  ASSERT(!supertype.IsNull());
  ASSERT(!subtype.IsNull());

  // Now that AssertSubtype may be checking types only available at runtime,
  // we can't guarantee the supertype isn't the top type.
  if (supertype.IsTopType()) return;

  // The supertype or subtype may not be instantiated.
  if (AbstractType::InstantiateAndTestSubtype(
          &subtype, &supertype, instantiator_type_args, function_type_args)) {
    if (FLAG_trace_type_checks) {
      // The supertype and subtype are now instantiated. Subtype check passed.
      PrintSubtypeCheck(subtype, supertype, true);
    }
    return;
  }
  if (FLAG_trace_type_checks) {
    // The supertype and subtype are now instantiated. Subtype check failed.
    PrintSubtypeCheck(subtype, supertype, false);
  }

  // Throw a dynamic type error.
  const TokenPosition location = GetCallerLocation();
  Exceptions::CreateAndThrowTypeError(location, subtype, supertype, dst_name);
  UNREACHABLE();
}

// Allocate a new closure and initializes its function, length,
// flags, context, hash and entry point.
// Arg0: function.
// Arg1: length and flags.
// Arg2: context.
// Return value: newly allocated closure.
DEFINE_RUNTIME_ENTRY(AllocateClosure, 3) {
  const auto& function = Function::CheckedHandle(zone, arguments.ArgAt(0));
  const intptr_t length_and_flags =
      Smi::CheckedHandle(zone, arguments.ArgAt(1)).Value();
  const auto& context = Object::Handle(zone, arguments.ArgAt(2));
  const Closure& closure = Closure::Handle(
      zone, Closure::New(length_and_flags, SpaceForRuntimeAllocation()));
  closure.set_function(function);
  if (!context.IsNull()) {
    closure.SetRawContext(context);
  }
  arguments.SetReturn(closure);
  RuntimeAllocationEpilogue(thread);
}

// Allocate a new context large enough to hold the given number of variables.
// Arg0: number of variables.
// Return value: newly allocated context.
DEFINE_RUNTIME_ENTRY(AllocateContext, 1) {
  const Smi& num_variables = Smi::CheckedHandle(zone, arguments.ArgAt(0));
  const Context& context = Context::Handle(
      zone, Context::New(num_variables.Value(), SpaceForRuntimeAllocation()));
  arguments.SetReturn(context);
  RuntimeAllocationEpilogue(thread);
}

// Make a copy of the given context, including the values of the captured
// variables.
// Arg0: the context to be cloned.
// Return value: newly allocated context.
DEFINE_RUNTIME_ENTRY(CloneContext, 1) {
  const Context& ctx = Context::CheckedHandle(zone, arguments.ArgAt(0));
  Context& cloned_ctx = Context::Handle(
      zone, Context::New(ctx.num_variables(), SpaceForRuntimeAllocation()));
  cloned_ctx.set_parent(Context::Handle(zone, ctx.parent()));
  Object& inst = Object::Handle(zone);
  for (int i = 0; i < ctx.num_variables(); i++) {
    inst = ctx.At(i);
    cloned_ctx.SetAt(i, inst);
  }
  arguments.SetReturn(cloned_ctx);
  RuntimeAllocationEpilogue(thread);
}

// Allocate a new record instance.
// Arg0: record shape id.
// Return value: newly allocated record.
DEFINE_RUNTIME_ENTRY(AllocateRecord, 1) {
  const RecordShape shape(Smi::RawCast(arguments.ArgAt(0)));
  const Record& record =
      Record::Handle(zone, Record::New(shape, SpaceForRuntimeAllocation()));
  arguments.SetReturn(record);
  RuntimeAllocationEpilogue(thread);
}

// Allocate a new small record instance and initialize its fields.
// Arg0: record shape id.
// Arg1-Arg3: field values.
// Return value: newly allocated record.
DEFINE_RUNTIME_ENTRY(AllocateSmallRecord, 4) {
  const RecordShape shape(Smi::RawCast(arguments.ArgAt(0)));
  const auto& value0 = Instance::CheckedHandle(zone, arguments.ArgAt(1));
  const auto& value1 = Instance::CheckedHandle(zone, arguments.ArgAt(2));
  const auto& value2 = Instance::CheckedHandle(zone, arguments.ArgAt(3));
  const Record& record =
      Record::Handle(zone, Record::New(shape, SpaceForRuntimeAllocation()));
  const intptr_t num_fields = shape.num_fields();
  ASSERT(num_fields == 2 || num_fields == 3);
  record.SetFieldAt(0, value0);
  record.SetFieldAt(1, value1);
  if (num_fields > 2) {
    record.SetFieldAt(2, value2);
  }
  arguments.SetReturn(record);
  RuntimeAllocationEpilogue(thread);
}

// Allocate a SuspendState object.
// Arg0: frame size.
// Arg1: existing SuspendState object or function data.
// Return value: newly allocated object.
// No lazy deopt: the various suspend stubs need to save the real pc, not the
// lazy deopt stub entry, for pointer visiting of the suspend state to work. The
// resume stubs will do a check for disabled code.
DEFINE_RUNTIME_ENTRY_NO_LAZY_DEOPT(AllocateSuspendState, 2) {
  const intptr_t frame_size =
      Smi::CheckedHandle(zone, arguments.ArgAt(0)).Value();
  const Object& previous_state = Object::Handle(zone, arguments.ArgAt(1));
  SuspendState& result = SuspendState::Handle(zone);
  if (previous_state.IsSuspendState()) {
    const auto& suspend_state = SuspendState::Cast(previous_state);
    const auto& function_data =
        Instance::Handle(zone, suspend_state.function_data());
    ObjectStore* object_store = thread->isolate_group()->object_store();
    if (function_data.GetClassId() ==
        Class::Handle(zone, object_store->async_star_stream_controller())
            .id()) {
      // Reset _AsyncStarStreamController.asyncStarBody to null in order
      // to create a new callback closure during next yield.
      // The new callback closure will capture the reallocated SuspendState.
      //
      // Caveat: can't use [SetField] here because it will try to take program
      // lock (to update the state of guarded cid) and that requires us to
      // be at safepoint which permits lazy deopt. Instead bypass
      // field guard by making sure that guarded_cid allows our store here.
      // (See ObjectStore::InitKnownObjects which initializes it).
      function_data.SetFieldWithoutFieldGuard(
          Field::Handle(
              zone,
              object_store->async_star_stream_controller_async_star_body()),
          Object::null_object());
    }
    result = SuspendState::New(frame_size, function_data,
                               SpaceForRuntimeAllocation());
    if (function_data.GetClassId() ==
        Class::Handle(zone, object_store->sync_star_iterator_class()).id()) {
      // Refresh _SyncStarIterator._state with the new SuspendState object.
      //
      // Caveat: can't use [SetField] here because it will try to take program
      // lock (to update the state of guarded cid) and that requires us to
      // be at safepoint which permits lazy deopt. Instead bypass
      // field guard by making sure that guarded_cid allows our store here.
      // (See ObjectStore::InitKnownObjects which initializes it).
      function_data.SetFieldWithoutFieldGuard(
          Field::Handle(zone, object_store->sync_star_iterator_state()),
          result);
    }
  } else {
    result = SuspendState::New(frame_size, Instance::Cast(previous_state),
                               SpaceForRuntimeAllocation());
  }
  arguments.SetReturn(result);
  RuntimeAllocationEpilogue(thread);
}

// Makes a copy of the given SuspendState object, including the payload frame.
// Arg0: the SuspendState object to be cloned.
// Return value: newly allocated object.
DEFINE_RUNTIME_ENTRY(CloneSuspendState, 1) {
  const SuspendState& src =
      SuspendState::CheckedHandle(zone, arguments.ArgAt(0));
  const SuspendState& dst = SuspendState::Handle(
      zone, SuspendState::Clone(thread, src, SpaceForRuntimeAllocation()));
  arguments.SetReturn(dst);
  RuntimeAllocationEpilogue(thread);
}

// Allocate a new SubtypeTestCache for use in interpreted implicit setters.
// Return value: newly allocated SubtypeTestCache.
DEFINE_RUNTIME_ENTRY(AllocateSubtypeTestCache, 0) {
#if defined(DART_DYNAMIC_MODULES)
  const auto& cache = SubtypeTestCache::Handle(
      zone, SubtypeTestCache::New(SubtypeTestCache::kMaxInputs));
  arguments.SetReturn(cache);
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES)
}

// Invoke field getter before dispatch.
// Arg0: instance.
// Arg1: field name (may be demangled during call).
// Return value: field value.
DEFINE_RUNTIME_ENTRY(GetFieldForDispatch, 2) {
#if defined(DART_DYNAMIC_MODULES)
  const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  String& name = String::CheckedHandle(zone, arguments.ArgAt(1));
  const Class& receiver_class = Class::Handle(zone, receiver.clazz());
  if (Function::IsDynamicInvocationForwarderName(name)) {
    name = Function::DemangleDynamicInvocationForwarderName(name);
    arguments.SetArgAt(1, name);  // Reflect change in arguments.
  }
  const String& getter_name = String::Handle(zone, Field::GetterName(name));
  const int kTypeArgsLen = 0;
  const int kNumArguments = 1;
  ArgumentsDescriptor args_desc(Array::Handle(
      zone, ArgumentsDescriptor::NewBoxed(kTypeArgsLen, kNumArguments)));
  const Function& getter = Function::Handle(
      zone, Resolver::ResolveDynamicForReceiverClass(
                receiver_class, getter_name, args_desc, /*allow_add=*/true));
  ASSERT(!getter.IsNull());  // An InvokeFieldDispatcher function was created.
  const Array& args = Array::Handle(zone, Array::New(kNumArguments));
  args.SetAt(0, receiver);
  const Object& result =
      Object::Handle(zone, DartEntry::InvokeFunction(getter, args));
  ThrowIfError(result);
  arguments.SetReturn(result);
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES)
}

// Converts arguments descriptor passed to an implicit closure
// into an arguments descriptor for the target function.
// Arg0: implicit closure arguments descriptor
// Arg1: target function
// Arg2: new type args length
// Return value: target arguments descriptor
DEFINE_RUNTIME_ENTRY(AdjustArgumentsDesciptorForImplicitClosure, 3) {
#if defined(DART_DYNAMIC_MODULES)
  const auto& descriptor = Array::CheckedHandle(zone, arguments.ArgAt(0));
  const auto& target = Function::CheckedHandle(zone, arguments.ArgAt(1));
  intptr_t type_args_len = Smi::CheckedHandle(zone, arguments.ArgAt(2)).Value();

  const ArgumentsDescriptor args_desc(descriptor);
  intptr_t num_arguments = args_desc.Count();

  if (target.is_static()) {
    // Drop closure receiver.
    --num_arguments;
  } else {
    if (target.IsGenerativeConstructor()) {
      // Type arguments are not passed to a generative constructor.
      type_args_len = 0;
    }
  }

  const auto& optional_arguments_names =
      Array::Handle(zone, args_desc.GetArgumentNames());
  const auto& result = Array::Handle(
      zone, ArgumentsDescriptor::NewBoxed(type_args_len, num_arguments,
                                          optional_arguments_names));
  arguments.SetReturn(result);
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES)
}

// Converts type arguments passed to a constructor tear-off
// into an instance type arguments.
// Arg0: class to allocate
// Arg1: type arguments
// Return value: instance type arguments
DEFINE_RUNTIME_ENTRY(ConvertToInstanceTypeArguments, 2) {
#if defined(DART_DYNAMIC_MODULES)
  const auto& cls = Class::CheckedHandle(zone, arguments.ArgAt(0));
  const auto& type_args =
      TypeArguments::CheckedHandle(zone, arguments.ArgAt(1));
  const auto& result = TypeArguments::Handle(
      zone, cls.GetInstanceTypeArguments(thread, type_args));
  arguments.SetReturn(result);
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES)
}

// Check that arguments are valid for the given closure.
// Arg0: closure
// Arg1: arguments descriptor
// Return value: whether the arguments are valid
DEFINE_RUNTIME_ENTRY(ClosureArgumentsValid, 2) {
#if defined(DART_DYNAMIC_MODULES)
  const auto& closure = Closure::CheckedHandle(zone, arguments.ArgAt(0));
  const auto& descriptor = Array::CheckedHandle(zone, arguments.ArgAt(1));

  const auto& function = Function::Handle(zone, closure.function());
  const ArgumentsDescriptor args_desc(descriptor);
  if (!function.AreValidArguments(args_desc, nullptr)) {
    arguments.SetReturn(Bool::False());
  } else if (!closure.IsGeneric() && args_desc.TypeArgsLen() > 0) {
    // The arguments may be valid for the closure function itself, but if the
    // closure has delayed type arguments, no type arguments should be provided.
    arguments.SetReturn(Bool::False());
  } else {
    arguments.SetReturn(Bool::True());
  }
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES)
}

// Resolve 'call' function of receiver.
// Arg0: receiver (not a closure).
// Arg1: arguments descriptor
// Return value: 'call' function'.
DEFINE_RUNTIME_ENTRY(ResolveCallFunction, 2) {
#if defined(DART_DYNAMIC_MODULES)
  const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const Array& descriptor = Array::CheckedHandle(zone, arguments.ArgAt(1));
  ArgumentsDescriptor args_desc(descriptor);
  ASSERT(!receiver.IsClosure());  // Interpreter tests for closure.
  Class& cls = Class::Handle(zone, receiver.clazz());
  Function& call_function = Function::Handle(
      zone,
      Resolver::ResolveDynamicForReceiverClass(cls, Symbols::call(), args_desc,
                                               /*allow_add=*/false));
  arguments.SetReturn(call_function);
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES)
}

// Resolve external method call from the interpreter.
// Arg0: function.
// Arg1: pool index to store resolved trampoline and native function.
DEFINE_RUNTIME_ENTRY(ResolveExternalCall, 2) {
#if defined(DART_DYNAMIC_MODULES)
  const auto& function = Function::CheckedHandle(zone, arguments.ArgAt(0));
  const intptr_t pool_index =
      Smi::CheckedHandle(zone, arguments.ArgAt(1)).Value();

  const Class& cls = Class::Handle(zone, function.Owner());
  const Library& library = Library::Handle(zone, cls.library());

  Dart_NativeEntryResolver resolver = library.native_entry_resolver();
  bool is_bootstrap_native = Bootstrap::IsBootstrapResolver(resolver);

  const String& native_name = String::Handle(zone, function.native_name());
  ASSERT(!native_name.IsNull());

  const intptr_t num_params =
      NativeArguments::ParameterCountForResolution(function);
  bool is_auto_scope = true;
  const NativeFunction target_function = NativeEntry::ResolveNative(
      library, native_name, num_params, &is_auto_scope);
  if (target_function == nullptr) {
    const auto& error = Error::Handle(LanguageError::NewFormatted(
        Error::Handle(),  // No previous error.
        Script::Handle(function.script()), function.token_pos(),
        Report::AtLocation, Report::kError, Heap::kOld,
        "native function '%s' (%" Pd " arguments) cannot be found",
        native_name.ToCString(), num_params));
    Exceptions::PropagateError(error);
  }

  NativeFunctionWrapper trampoline;
  if (is_bootstrap_native) {
    trampoline = NativeEntry::BootstrapNativeCallWrapper;
  } else if (is_auto_scope) {
    trampoline = NativeEntry::AutoScopeNativeCallWrapper;
  } else {
    trampoline = NativeEntry::NoScopeNativeCallWrapper;
  }

  const auto& bytecode = Bytecode::Handle(zone, function.GetBytecode());
  const auto& pool = ObjectPool::Handle(zone, bytecode.object_pool());
  pool.SetRawValueAt(pool_index, reinterpret_cast<uword>(trampoline));
  pool.SetRawValueAt(pool_index + 1, reinterpret_cast<uword>(target_function));
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES)
}

#if defined(DART_DYNAMIC_MODULES)

#if !defined(DART_PRECOMPILED_RUNTIME)

struct FfiCallArguments {
  uword stack_area;
  uword stack_area_end;
  uword cpu_registers[kNumberOfCpuRegisters];
  uword fpu_registers[kNumberOfFpuRegisters];
  uword target;

  const char* ToCString(Zone* zone, bool print_stack_as_words = false) const {
    ZoneTextBuffer buffer(zone);
    buffer.AddString("FfiCallArguments {\n");
    buffer.Printf("  Target: %#" Px "\n", target);
    buffer.AddString("  CPU registers:\n");
    for (intptr_t i = 0; i < kNumberOfCpuRegisters; ++i) {
      const char* name =
          RegisterNames::RegisterAbiName(static_cast<Register>(i));
      buffer.Printf("   %5s: %#" Px "\n", name, cpu_registers[i]);
    }
    buffer.AddString("  FPU registers:\n");
    for (intptr_t i = 0; i < kNumberOfFpuRegisters; ++i) {
      const char* name =
          RegisterNames::FpuRegisterName(static_cast<FpuRegister>(i));
      buffer.Printf("   %5s: %#" Px "\n", name, fpu_registers[i]);
    }
    const intptr_t stack_size = stack_area_end - stack_area;
    buffer.Printf("  Stack area (size %" Pu "):", stack_size);
    const intptr_t kIncrement = print_stack_as_words ? kWordSize : 1;
    const intptr_t kBytesPerLine = print_stack_as_words ? kWordSize : 16;
    for (intptr_t i = 0; i < stack_size; i += kIncrement) {
      if ((i % kBytesPerLine) == 0) {
        buffer.Printf("\n    %#" Px ": ", stack_area + i);
      } else if ((i % kWordSize) == 0) {
        buffer.AddString("  ");
      }
      if (print_stack_as_words) {
        buffer.Printf("%#" Px "",
                      *reinterpret_cast<const uword*>(stack_area + i));
      } else {
        buffer.Printf("%02.2x",
                      *reinterpret_cast<const uint8_t*>(stack_area + i));
      }
    }
    buffer.AddString("\n}\n");
    return buffer.buffer();
  }
};

#if defined(HOST_ARCH_ARM64)
extern "C" void FfiCallTrampoline(FfiCallArguments* args);
#else
extern "C" typedef void (*ffiCallTrampoline)(FfiCallArguments* args);

void FfiCallTrampoline(FfiCallArguments* args) {
#if defined(HOST_ARCH_X64)
  reinterpret_cast<ffiCallTrampoline>(
      StubCode::FfiCallTrampoline().EntryPoint())(args);
#else
  UNIMPLEMENTED();
#endif
}
#endif

static int64_t TruncateFfiInt(int64_t value,
                              compiler::ffi::PrimitiveType type,
                              bool is_return) {
#if defined(HOST_ARCH_RISCV64)
  // 64-bit RISC-V represents C uint32 as sign-extended to 64 bits.
  if (!is_return && (type == compiler::ffi::kUint32)) {
    return static_cast<int32_t>(static_cast<uint32_t>(value));
  }
#endif
  switch (type) {
    case compiler::ffi::kInt8:
      return static_cast<int8_t>(value);
    case compiler::ffi::kUint8:
      return static_cast<uint8_t>(value);
    case compiler::ffi::kInt16:
      return static_cast<int16_t>(value);
    case compiler::ffi::kUint16:
      return static_cast<uint16_t>(value);
    case compiler::ffi::kInt32:
      return static_cast<int32_t>(value);
    case compiler::ffi::kUint32:
      return static_cast<uint32_t>(value);
    case compiler::ffi::kInt64:
    case compiler::ffi::kUint64:
      return value;
    default:
      UNREACHABLE();
  }
}

static void* GetDataAddress(const Instance& inst, intptr_t offset_in_bytes) {
  if (inst.IsTypedDataBase()) {
    return TypedDataBase::Cast(inst).DataAddr(offset_in_bytes);
  } else if (inst.IsPointer()) {
    return reinterpret_cast<void*>(
        reinterpret_cast<uint8_t*>(Pointer::Cast(inst).NativeAddress()) +
        offset_in_bytes);
  } else {
    UNIMPLEMENTED();
  }
}

static ObjectPtr FfiConvertPrimitiveToDart(
    Thread* thread,
    const compiler::ffi::BaseMarshaller& marshaller,
    intptr_t arg_index,
    uword value) {
  const auto& loc = marshaller.Location(arg_index);
  const bool is_return = marshaller.ArgumentIndexIsReturn(arg_index);
  if (marshaller.IsPointerPointer(arg_index)) {
    return Pointer::New(value);
  } else if (marshaller.IsTypedDataPointer(arg_index)) {
    UNREACHABLE();  // Only supported for FFI call arguments.
  } else if (marshaller.IsCompoundPointer(arg_index)) {
    UNREACHABLE();  // Only supported for FFI call arguments.
  } else if (marshaller.IsHandleCType(arg_index)) {
    return reinterpret_cast<LocalHandle*>(value)->ptr();
  } else if (marshaller.IsBool(arg_index)) {
    value = TruncateFfiInt(
        value, loc.payload_type().AsPrimitive().representation(), is_return);
    return Bool::Get(value != 0).ptr();
  } else if (marshaller.IsVoid(arg_index)) {
    return Object::null();
  } else {
    auto const rep = marshaller.RepInDart(arg_index);
    if (RepresentationUtils::IsUnboxedInteger(rep)) {
      value = TruncateFfiInt(
          value, loc.payload_type().AsPrimitive().representation(), is_return);
      return Integer::New(value);
    } else if (rep == kUnboxedDouble) {
      return Double::New(bit_cast<double, uword>(value));
    } else if (rep == kUnboxedFloat) {
      float f = bit_cast<float, uint32_t>(static_cast<uint32_t>(value));
      return Double::New(static_cast<double>(f));
    } else {
      UNREACHABLE();
    }
  }
}

static uword FfiConvertPrimitiveToNative(
    Thread* thread,
    const compiler::ffi::BaseMarshaller& marshaller,
    intptr_t arg_index,
    const Object& obj) {
  Zone* const zone = thread->zone();
  // Error values should only be possible for returns in callbacks.
  ASSERT(!obj.IsError() || arg_index == compiler::ffi::kResultIndex);
  // First, handle the cases that allow Error values the same as in
  // FlowGraphBuilder::BuildGraphOfSyncFfiCallback.
  if (marshaller.IsVoid(arg_index)) {
    ASSERT_EQUAL(arg_index, compiler::ffi::kResultIndex);
    // Ignore the return value or exception and return 0.
    if (obj.IsError()) {
      PRINT_IF_TRACING_INTERPRETER(
          "suppressing exception due to void return\n");
    }
    return 0;
  } else if (marshaller.IsPointerPointer(arg_index)) {
    if (obj.IsError()) {
      PRINT_IF_TRACING_INTERPRETER("returning nullptr instead\n");
      return reinterpret_cast<uword>(nullptr);
    }
    return Pointer::Cast(obj).NativeAddress();
  } else if (marshaller.IsHandleCType(arg_index)) {
    // No special handling needed for Errors.
    ApiLocalScope* scope = thread->api_top_scope();
    LocalHandle* handle = scope->local_handles()->AllocateHandle();
    handle->set_ptr(obj.ptr());
    return reinterpret_cast<uword>(handle);
  }
  // Now handle all cases where Error objects are not allowed.
  ASSERT(obj.IsInstance());
  if (marshaller.IsTypedDataPointer(arg_index) ||
      marshaller.IsCompoundPointer(arg_index)) {
    // Should only happen for FFI calls (e.g., as arguments, not returns).
    ASSERT(!marshaller.ArgumentIndexIsReturn(arg_index));

    // Fields needed for working with compound values.
    auto* const object_store = thread->isolate_group()->object_store();
    const auto& typed_data_field =
        Field::Handle(zone, object_store->compound_typed_data_base_field());
    const auto& offset_in_bytes_field =
        Field::Handle(zone, object_store->compound_offset_in_bytes_field());

    auto& compound_contents = Instance::CheckedHandle(zone, obj.ptr());
    intptr_t offset_in_bytes = 0;
    if (marshaller.IsCompoundPointer(arg_index)) {
      offset_in_bytes = Smi::Value(
          Smi::RawCast(compound_contents.GetField(offset_in_bytes_field)));
      compound_contents ^= compound_contents.GetField(typed_data_field);
    }

    // Object holding the contents should not be moved by GC. For leaf calls,
    // the native code is run in a NoSafepointScope, and only Pointers are
    // allowed for non-leaf calls.
    ASSERT(thread->no_safepoint_scope_depth() > 0 ||
           compound_contents.IsPointer());
    return reinterpret_cast<uword>(
        GetDataAddress(compound_contents, offset_in_bytes));
  } else if (marshaller.IsBool(arg_index)) {
    return static_cast<uword>(Bool::Cast(obj).value());
  } else {
    const auto rep = marshaller.RepInDart(arg_index);
    if (RepresentationUtils::IsUnboxedInteger(rep)) {
      const auto& loc = marshaller.Location(arg_index);
      const bool is_return = marshaller.ArgumentIndexIsReturn(arg_index);
      return TruncateFfiInt(Integer::Cast(obj).Value(),
                            loc.payload_type().AsPrimitive().representation(),
                            is_return);
    } else if (rep == kUnboxedDouble) {
      return bit_cast<uint64_t, double>(Double::Cast(obj).value());
    } else if (rep == kUnboxedFloat) {
      return bit_cast<uint32_t, float>(
          static_cast<float>(Double::Cast(obj).value()));
    } else {
      UNREACHABLE();
    }
  }
}

static void SetLocationInFfiCallArguments(
    Thread* thread,
    FfiCallArguments* args,
    const compiler::ffi::NativeLocation& loc,
    uword value) {
  if (loc.IsRegisters()) {
    ASSERT_EQUAL(loc.AsRegisters().num_regs(), 1);
    const auto dst_reg = loc.AsRegisters().reg_at(0);
    ASSERT((dst_reg >= 0) && (dst_reg < kNumberOfCpuRegisters));
    args->cpu_registers[dst_reg] = value;
  } else if (loc.IsFpuRegisters()) {
    const auto dst_reg = loc.AsFpuRegisters().fpu_reg();
    ASSERT((dst_reg >= 0) && (dst_reg < kNumberOfFpuRegisters));
    args->fpu_registers[dst_reg] = value;
  } else if (loc.IsStack()) {
    const intptr_t offset = loc.AsStack().offset_in_bytes();
    ASSERT(offset >= 0);
    ASSERT(args->stack_area + offset + kWordSize <= args->stack_area_end);
    *reinterpret_cast<uword*>(args->stack_area + offset) = value;
  } else {
    FATAL("Unimplemented location %s", loc.ToCString(thread->zone()));
  }
}

static void PassFfiCallCompoundArgumentPortion(
    Thread* thread,
    FfiCallArguments* args,
    const compiler::ffi::NativeLocation& loc,
    const Instance& compound_contents,
    intptr_t offset_in_bytes) {
  auto const size = loc.payload_type().SizeInBytes();
  if (loc.IsMultiple()) {
    // Copy from the TypedData to from a list of native locations.
    const auto& multiple = loc.AsMultiple();
    for (intptr_t i = 0, n = multiple.locations().length(); i < n; ++i) {
      const auto& portion = *multiple.locations().At(i);
      PassFfiCallCompoundArgumentPortion(thread, args, portion,
                                         compound_contents, offset_in_bytes);
      offset_in_bytes += portion.payload_type().SizeInBytes();
    }
  } else if (loc.IsStack()) {
    const intptr_t offset = loc.AsStack().offset_in_bytes();
    ASSERT((offset >= 0) &&
           (args->stack_area + offset + size <= args->stack_area_end));
    auto* const dst = reinterpret_cast<uint8_t*>(args->stack_area) + offset;
    NoSafepointScope scope;
    auto* const src = GetDataAddress(compound_contents, offset_in_bytes);
    memcpy(dst, src, size);  // NOLINT
  } else {
    if (!loc.payload_type().IsPrimitive()) {
      UNIMPLEMENTED();
    }
    ASSERT(size <= kWordSize);
    uword value;
    {
      NoSafepointScope scope;
      auto* const src = GetDataAddress(compound_contents, offset_in_bytes);
      memcpy(&value, src, size);  // NOLINT
    }
    SetLocationInFfiCallArguments(thread, args, loc, value);
  }
}

static void PassFfiCallArguments(
    Thread* thread,
    const compiler::ffi::CallMarshaller& marshaller,
    ObjectPtr* argv,
    FfiCallArguments* args,
    bool is_leaf) {
  Zone* zone = thread->zone();
  // Fields needed for working with compound values.
  auto* const object_store = thread->isolate_group()->object_store();
  const auto& typed_data_field =
      Field::Handle(zone, object_store->compound_typed_data_base_field());
  const auto& offset_in_bytes_field =
      Field::Handle(zone, object_store->compound_offset_in_bytes_field());
  auto& compound_contents = Instance::Handle(zone);
  auto& arg = Object::Handle(zone);
  for (intptr_t i = 0; i < marshaller.num_args(); ++i) {
    arg = argv[i];
    const auto& loc = marshaller.Location(i);
    PRINT_IF_TRACING_INTERPRETER("Arg %" Pd " -> %s\n", i, loc.ToCString());
    if (marshaller.IsCompoundCType(i)) {
      compound_contents ^= Instance::Cast(arg).GetField(typed_data_field);
      intptr_t offset_in_bytes = Smi::Value(
          Smi::RawCast(Instance::Cast(arg).GetField(offset_in_bytes_field)));
      if (loc.IsPointerToMemory()) {
        const auto& arg_loc = loc.AsPointerToMemory();
        auto const size = arg_loc.payload_type().SizeInBytes();
        auto const stack_offset = marshaller.PassByPointerStackOffset(i);
        ASSERT(stack_offset >= 0);
        ASSERT(args->stack_area + stack_offset + size <= args->stack_area_end);
        auto* const dst =
            reinterpret_cast<void*>(args->stack_area + stack_offset);
        // First copy the contents of the struct to the stack area.
        {
          NoSafepointScope scope;
          auto* const src = GetDataAddress(compound_contents, offset_in_bytes);
          memcpy(dst, src, size);  // NOLINT
        }
        // Then copy the pointer to that memory to the expected location.
        const auto& ptr_loc = arg_loc.pointer_location();
        const uword ptr = reinterpret_cast<uword>(dst);
        SetLocationInFfiCallArguments(thread, args, ptr_loc, ptr);
      } else {
        PassFfiCallCompoundArgumentPortion(thread, args, loc, compound_contents,
                                           offset_in_bytes);
      }
    } else {
      if (!loc.payload_type().IsPrimitive()) {
        UNIMPLEMENTED();
      }
      ASSERT(loc.payload_type().SizeInBytes() <= kWordSize);
      const uword value =
          FfiConvertPrimitiveToNative(thread, marshaller, i, arg);
      SetLocationInFfiCallArguments(thread, args, loc, value);
    }
  }

#if defined(TARGET_ARCH_X64)
  if (marshaller.contains_varargs() &&
      CallingConventions::kVarArgFpuRegisterCount != kNoRegister) {
    // TODO(http://dartbug.com/38578): Use the number of used FPU registers.
    args->cpu_registers[CallingConventions::kVarArgFpuRegisterCount] =
        CallingConventions::kFpuArgumentRegisters;
  }
#endif  // defined(TARGET_ARCH_X64)

  if (marshaller.ReturnsCompound()) {
    const intptr_t arg_index = compiler::ffi::kResultIndex;
    const auto& loc = marshaller.Location(arg_index);
    if (loc.IsPointerToMemory()) {
      // Pass a pointer to the space allocated in stack_area to native code.
      const auto& ptr_loc = loc.AsPointerToMemory().pointer_location();
      const intptr_t offset = marshaller.PassByPointerStackOffset(arg_index);
      const intptr_t size = loc.payload_type().SizeInBytes();
      ASSERT(offset >= 0);
      ASSERT(args->stack_area + offset + size <= args->stack_area_end);
      const uword ptr = args->stack_area + offset;
      SetLocationInFfiCallArguments(thread, args, ptr_loc, ptr);
    }
  }
  PRINT_IF_TRACING_INTERPRETER("%s", args->ToCString(zone));
}

static uword RetrieveLocationFromFfiCallArguments(
    Thread* thread,
    const FfiCallArguments& args,
    const compiler::ffi::NativeLocation& loc) {
  if (loc.IsRegisters()) {
    ASSERT_EQUAL(loc.AsRegisters().num_regs(), 1);
    const auto src_reg = loc.AsRegisters().reg_at(0);
    ASSERT((src_reg >= 0) && (src_reg < kNumberOfCpuRegisters));
    return args.cpu_registers[src_reg];
  } else if (loc.IsFpuRegisters()) {
    const auto src_reg = loc.AsFpuRegisters().fpu_reg();
    ASSERT((src_reg >= 0) && (src_reg < kNumberOfFpuRegisters));
    return args.fpu_registers[src_reg];
  } else {
    FATAL("Unimplemented location %s", loc.ToCString(thread->zone()));
  }
}

static ObjectPtr ReceiveFfiCallResult(
    Thread* thread,
    const compiler::ffi::CallMarshaller& marshaller,
    const FfiCallArguments& args) {
  auto* const zone = thread->zone();
  const intptr_t arg_index = compiler::ffi::kResultIndex;
  const auto& loc = marshaller.Location(arg_index);
  PRINT_IF_TRACING_INTERPRETER("%s", args.ToCString(zone));
  PRINT_IF_TRACING_INTERPRETER("Return value <- %s\n", loc.ToCString());
  if (marshaller.IsCompoundCType(arg_index)) {
    const auto& compound_contents = TypedData::Handle(
        zone, TypedData::New(kTypedDataUint8ArrayCid,
                             marshaller.CompoundReturnSizeInBytes()));
    if (loc.IsPointerToMemory()) {
      auto const size = loc.payload_type().SizeInBytes();
      const intptr_t offset = marshaller.PassByPointerStackOffset(arg_index);
      ASSERT((offset >= 0) &&
             (args.stack_area + offset + size <= args.stack_area_end));
      auto* const src = reinterpret_cast<const void*>(args.stack_area + offset);
      NoSafepointScope scope;
      memcpy(compound_contents.DataAddr(0), src, size);  // NOLINT
    } else {
      // Copy to the TypedData buffer from a list of native locations.
      ASSERT(loc.IsMultiple());
      const auto& multiple = loc.AsMultiple();
      intptr_t offset_in_bytes = 0;

      for (intptr_t i = 0, n = multiple.locations().length(); i < n; ++i) {
        const auto& portion = *multiple.locations().At(i);
        // Only structs small enough to fit in a CPU + FPU register combo
        // or two FPU registers are sent as multiple locations.
        if (!portion.payload_type().IsPrimitive()) {
          UNIMPLEMENTED();
        }
        auto const size = portion.payload_type().SizeInBytes();
        ASSERT(size <= kWordSize);
        const uword value =
            RetrieveLocationFromFfiCallArguments(thread, args, portion);
        NoSafepointScope scope;
        auto* dst = compound_contents.DataAddr(offset_in_bytes);
        memcpy(dst, &value, size);  // NOLINT
        offset_in_bytes += size;
      }
    }
    // Now that the contents have been collected, time to install the
    // appropriate wrapper.
    auto* const object_store = thread->isolate_group()->object_store();
    const auto& typed_data_field =
        Field::Handle(zone, object_store->compound_typed_data_base_field());
    const auto& offset_in_bytes_field =
        Field::Handle(zone, object_store->compound_offset_in_bytes_field());

    const auto& compound_type =
        AbstractType::Handle(zone, marshaller.CType(arg_index));
    const auto& compound_sub_class =
        Class::Handle(zone, compound_type.type_class());
    compound_sub_class.EnsureIsFinalized(thread);

    const auto& wrapper =
        Instance::Handle(zone, Instance::New(compound_sub_class));
    wrapper.SetField(typed_data_field, compound_contents);
    wrapper.SetField(offset_in_bytes_field, Smi::Handle(Smi::New(0)));

    return wrapper.ptr();
  } else {
    const uword value = RetrieveLocationFromFfiCallArguments(thread, args, loc);
    return FfiConvertPrimitiveToDart(thread, marshaller, arg_index, value);
  }
}

static uword ResolveFfiNativeTarget(Thread* thread, const Instance& native) {
  Zone* zone = thread->zone();
  const auto& native_class = Class::Handle(zone, native.clazz());
  ASSERT(String::Handle(native_class.UserVisibleName())
             .Equals(Symbols::FfiNative()));
  const auto& symbol_field = Field::Handle(
      zone, native_class.LookupInstanceFieldAllowPrivate(Symbols::symbol()));
  ASSERT(!symbol_field.IsNull());
  const auto& asset_id_field = Field::Handle(
      zone, native_class.LookupInstanceFieldAllowPrivate(Symbols::assetId()));
  ASSERT(!asset_id_field.IsNull());
  const auto& symbol =
      String::Handle(zone, String::RawCast(native.GetField(symbol_field)));
  const auto& asset_id =
      String::Handle(zone, String::RawCast(native.GetField(asset_id_field)));
  const auto& type_args =
      TypeArguments::Handle(zone, native.GetTypeArguments());
  ASSERT(type_args.Length() == 1);
  const auto& native_type = AbstractType::Handle(zone, type_args.TypeAt(0));
  intptr_t arg_n;
  if (native_type.IsFunctionType()) {
    const auto& native_function_type = FunctionType::Cast(native_type);
    arg_n = native_function_type.NumParameters() -
            native_function_type.num_implicit_parameters();
  } else {
    // We're looking up the address of a native field.
    arg_n = 0;
  }
  const auto& ffi_resolver = Function::ZoneHandle(
      zone, thread->isolate_group()->object_store()->ffi_resolver_function());
  const auto& args = Array::Handle(zone, Array::New(3));
  args.SetAt(0, asset_id);
  args.SetAt(1, symbol);
  args.SetAt(2, Smi::Handle(zone, Smi::New(arg_n)));
  const auto& result =
      Object::Handle(zone, DartEntry::InvokeFunction(ffi_resolver, args));
  ThrowIfError(result);
  auto const address = static_cast<uword>(Integer::Cast(result).Value());
  PRINT_IF_TRACING_INTERPRETER("resolved (%s, %s, %s) to %#" Px "\n",
                               symbol.ToCString(), asset_id.ToCString(),
                               native_type.ToCString(), address);
  return address;
}
#endif  // !defined(DART_PRECOMPILED_RUNTIME)
#endif  // defined(DART_DYNAMIC_MODULES)

// Resolve a native function from the interpreter.
// Arg0: constant instance of dart:ffi::Native.
// Arg1: function containing the ResolveNativeFunction instruction.
// Arg2: object pool index to store resolved target
//
// This method does not return anything, but instead caches the resolved
// entry point in the object pool entry.
DEFINE_RUNTIME_ENTRY(ResolveNativeFunction, 3) {
#if defined(DART_DYNAMIC_MODULES) && !defined(DART_PRECOMPILED_RUNTIME)
  const auto& instance = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const auto& function = Function::CheckedZoneHandle(zone, arguments.ArgAt(1));
  const intptr_t pool_index =
      Smi::CheckedHandle(zone, arguments.ArgAt(2)).Value();

  const auto& bytecode = Bytecode::Handle(zone, function.GetBytecode());
  const auto& pool = ObjectPool::Handle(zone, bytecode.object_pool());
  // The interpreter should only call this once for a particular
  // ResolveNativeFunction instruction, as the resolver is idempotent
  // and so the result can be cached.
  const uword target = ResolveFfiNativeTarget(thread, instance);
  ASSERT(target != 0);
  pool.SetRawValueAt(pool_index, target);
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES) && !defined(DART_PRECOMPILED_RUNTIME)
}

// Perform FFI call from the interpreter.
// Arg0: function.
// Arg1: constant pool index to store resolved target.
DEFINE_RUNTIME_ENTRY(FfiCall, 2) {
#if defined(DART_DYNAMIC_MODULES) && !defined(DART_PRECOMPILED_RUNTIME)
  const auto& function = Function::CheckedZoneHandle(zone, arguments.ArgAt(0));
  const intptr_t pool_index =
      Smi::CheckedHandle(zone, arguments.ArgAt(1)).Value();
  ASSERT(function.is_ffi_native() || function.IsFfiCallClosure());

  StackFrameIterator iterator(ValidationPolicy::kDontValidateFrames, thread,
                              StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* frame = iterator.NextFrame();
  ASSERT(frame != nullptr);
  ASSERT(frame->IsExitFrame());
  frame = iterator.NextFrame();
  ASSERT(frame != nullptr);
  ASSERT(frame->IsDartFrame());
  ASSERT(frame->is_interpreted());

  const uword fp = frame->fp();
  const uword sp = arguments.GetCallerSP();
  ASSERT_LESS_OR_EQUAL(fp, sp);
  ASSERT_LESS_OR_EQUAL(sp, frame->sp());
  MSAN_UNPOISON(reinterpret_cast<uint8_t*>(fp), sp - fp);

  ObjectPtr* argv = reinterpret_cast<ObjectPtr*>(sp);
  uword target;
  if (function.is_ffi_native()) {
    const auto& bytecode = Bytecode::Handle(zone, function.GetBytecode());
    const auto& pool = ObjectPool::Handle(zone, bytecode.object_pool());
    target = pool.RawValueAt(pool_index);
    if (target == 0) {
      PRINT_IF_TRACING_INTERPRETER("resolving FFI native %s\n",
                                   function.ToFullyQualifiedCString());
      const auto& annotation =
          Instance::Handle(zone, function.GetNativeAnnotation());
      target = ResolveFfiNativeTarget(thread, annotation);
      ASSERT(target != 0);
      pool.SetRawValueAt(pool_index, target);
    }
  } else {
    target = Pointer::CheckedHandle(zone, argv[-1]).NativeAddress();
  }

  const intptr_t first_argument_parameter_offset =
      function.IsFfiCallClosure() ? 1 : 0;
  const auto& c_signature =
      FunctionType::ZoneHandle(zone, function.FfiCSignature());
  const bool is_leaf = function.FfiIsLeaf();

  const char* error = nullptr;
  const auto marshaller_ptr = compiler::ffi::CallMarshaller::FromFunction(
      zone, function, first_argument_parameter_offset, c_signature, &error);
  // AbiSpecificTypes can have an incomplete mapping.
  if (error != nullptr) {
    const auto& language_error = Error::Handle(
        LanguageError::New(String::Handle(String::New(error, Heap::kOld)),
                           Report::kError, Heap::kOld));
    Report::LongJump(language_error);
  }

  RELEASE_ASSERT(marshaller_ptr != nullptr);
  const auto& marshaller = *marshaller_ptr;
  const intptr_t stack_area_size =
      Utils::RoundUp(marshaller.RequiredStackSpaceInBytes(), kWordSize * 2);
  uint8_t* stack_area = zone->Alloc<uint8_t>(stack_area_size);

  FfiCallArguments args;
  memset(&args, 0, sizeof(args));
  args.stack_area = reinterpret_cast<uword>(stack_area);
  args.stack_area_end = reinterpret_cast<uword>(stack_area + stack_area_size);
  args.target = target;

  argv = argv - first_argument_parameter_offset - marshaller.num_args();

  PRINT_IF_TRACING_INTERPRETER("calling native entry point %#" Px " for %s\n",
                               target, function.ToFullyQualifiedCString());
  if (is_leaf) {
    NoSafepointScope no_safepoint;
    PassFfiCallArguments(thread, marshaller, argv, &args, is_leaf);
    FfiCallTrampoline(&args);
  } else {
    PassFfiCallArguments(thread, marshaller, argv, &args, is_leaf);
    {
      TransitionVMToNative transition(thread);
      FfiCallTrampoline(&args);
    }
    // An FFI callback may have exited the isolate, but the UnwindError was
    // replaced with the callback's exceptional value on returning to native
    // code. Resume the unwinding process here. (See DLRT_ExitSafepoint.)
    if (thread->is_unwind_in_progress()) {
      thread->SetUnwindErrorInProgress(false);
      NoSafepointScope no_safepoint;
      Exceptions::PropagateError(Object::unwind_error());
    }
  }
  PRINT_IF_TRACING_INTERPRETER("returned from native entry point %#" Px "\n",
                               target);
  const auto& result =
      Object::Handle(zone, ReceiveFfiCallResult(thread, marshaller, args));
  ThrowIfError(result);
  arguments.SetReturn(result);
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES) && !defined(DART_PRECOMPILED_RUNTIME)
}

#if defined(HOST_ARCH_ARM64) && defined(DART_DYNAMIC_MODULES)
#if !defined(DART_PRECOMPILED_RUNTIME)
static void CopyLocationFromCallbackContextStack(
    void* dst,
    const CallbackContext& ctxt,
    const compiler::ffi::NativeStackLocation& loc) {
  auto* const sp = reinterpret_cast<const uint8_t*>(ctxt.sp);
  const intptr_t offset = loc.offset_in_bytes();
  const intptr_t size = loc.payload_type().SizeInBytes();
  memcpy(dst, sp + offset, size);  // NOLINT
}

static uword RetrieveLocationFromCallbackContext(
    Thread* thread,
    const CallbackContext& ctxt,
    intptr_t arg_index,
    const compiler::ffi::NativeLocation& loc) {
  const bool is_return = arg_index == compiler::ffi::kResultIndex;
  if (loc.IsRegisters()) {
    ASSERT_EQUAL(loc.AsRegisters().num_regs(), 1);
    const auto dst_reg = loc.AsRegisters().reg_at(0);
    ASSERT(dst_reg >= 0);
    if (is_return) {
      // The only use when exiting is to load the pointer to result struct.
      ASSERT_EQUAL(dst_reg,
                   CallingConventions::kPointerToReturnStructRegisterReturn);
      return ctxt.return_struct_pointer;
    }
    // If the pointer to return struct register overlaps with one of the
    // argument registers, then they're both set to the same value on entry.
    if (dst_reg == CallingConventions::kPointerToReturnStructRegisterCall) {
      return ctxt.return_struct_pointer;
    } else {
      ASSERT_LESS_OR_EQUAL(dst_reg, CallbackContext::kNumIntegerArguments - 1);
      return ctxt.integer_arguments[dst_reg];
    }
  } else if (loc.IsFpuRegisters()) {
    ASSERT(!is_return);  // FPU registers are only set, never read on exit.
    const auto dst_reg = loc.AsFpuRegisters().fpu_reg();
    ASSERT(dst_reg >= 0);
    ASSERT_LESS_OR_EQUAL(dst_reg, CallbackContext::kNumDoubleArguments - 1);
    return ctxt.double_arguments[dst_reg];
  } else if (loc.IsStack()) {
    ASSERT(!is_return);  // The stack is only set, never read on exit.
    ASSERT(loc.payload_type().SizeInBytes() <= kWordSize);
    uword result;
    CopyLocationFromCallbackContextStack(&result, ctxt, loc.AsStack());
    return result;
  } else {
    FATAL("Unimplemented location %s", loc.ToCString(thread->zone()));
  }
}

static ObjectPtr ReceiveFfiCallbackArgument(
    Thread* thread,
    const CallbackContext& ctxt,
    const compiler::ffi::CallbackMarshaller& marshaller,
    intptr_t arg_index) {
  ASSERT(!marshaller.IsVoid(arg_index));
  auto* const zone = thread->zone();
  const auto& loc = marshaller.Location(arg_index);
  PRINT_IF_TRACING_INTERPRETER("Arg %" Pd " <- %s\n", arg_index,
                               loc.ToCString());
  if (marshaller.IsCompoundCType(arg_index)) {
    const intptr_t size_in_bytes = loc.payload_type().SizeInBytes();
    const auto& compound_contents = TypedData::Handle(
        zone, TypedData::New(kTypedDataUint8ArrayCid, size_in_bytes));
    if (loc.IsPointerToMemory()) {
      const uword ptr = RetrieveLocationFromCallbackContext(
          thread, ctxt, arg_index, loc.AsPointerToMemory().pointer_location());
      auto* const src = reinterpret_cast<const void*>(ptr);
      NoSafepointScope scope;
      memcpy(compound_contents.DataAddr(0), src, size_in_bytes);  // NOLINT
    } else if (loc.IsStack()) {
      NoSafepointScope scope;
      CopyLocationFromCallbackContextStack(compound_contents.DataAddr(0), ctxt,
                                           loc.AsStack());
    } else {
      // Copy to the TypedData buffer from a list of native locations.
      ASSERT(loc.IsMultiple());
      const auto& multiple = loc.AsMultiple();
      intptr_t offset_in_bytes = 0;

      for (intptr_t i = 0, n = multiple.locations().length(); i < n; ++i) {
        const auto& portion = *multiple.locations().At(i);
        // Only structs small enough to fit in a CPU + FPU register combo
        // or two FPU registers are sent as multiple locations.
        if (!portion.payload_type().IsPrimitive()) {
          UNIMPLEMENTED();
        }
        auto const size = portion.payload_type().SizeInBytes();
        ASSERT(size <= kWordSize);
        const uword value = RetrieveLocationFromCallbackContext(
            thread, ctxt, arg_index, portion);
        NoSafepointScope scope;
        auto* dst = compound_contents.DataAddr(offset_in_bytes);
        memcpy(dst, &value, size);  // NOLINT
        offset_in_bytes += size;
      }
    }
    // Now that the contents have been collected, time to install the
    // appropriate wrapper.
    auto* const object_store = thread->isolate_group()->object_store();
    const auto& typed_data_field =
        Field::Handle(zone, object_store->compound_typed_data_base_field());
    const auto& offset_in_bytes_field =
        Field::Handle(zone, object_store->compound_offset_in_bytes_field());

    const auto& compound_type =
        AbstractType::Handle(zone, marshaller.CType(arg_index));
    const auto& compound_sub_class =
        Class::Handle(zone, compound_type.type_class());
    compound_sub_class.EnsureIsFinalized(thread);

    const auto& wrapper =
        Instance::Handle(zone, Instance::New(compound_sub_class));
    wrapper.SetField(typed_data_field, compound_contents);
    wrapper.SetField(offset_in_bytes_field, Smi::Handle(Smi::New(0)));

    return wrapper.ptr();
  } else {
    if (!loc.payload_type().IsPrimitive()) {
      FATAL("Unimplemented location %s", loc.ToCString(zone));
    }
    ASSERT(loc.payload_type().SizeInBytes() <= kWordSize);
    const uword value =
        RetrieveLocationFromCallbackContext(thread, ctxt, arg_index, loc);
    return FfiConvertPrimitiveToDart(thread, marshaller, arg_index, value);
  }
}

static void SetLocationInCallbackContext(
    Thread* thread,
    CallbackContext* ctxt,
    const compiler::ffi::NativeLocation& loc,
    uword value) {
  if (loc.IsRegisters()) {
    ASSERT_EQUAL(loc.AsRegisters().num_regs(), 1);
    const auto dst_reg = loc.AsRegisters().reg_at(0);
    ASSERT(dst_reg >= 0);
    ASSERT_LESS_OR_EQUAL(dst_reg, CallbackContext::kNumIntegerArguments - 1);
    ctxt->integer_arguments[dst_reg] = value;
  } else if (loc.IsFpuRegisters()) {
    const auto dst_reg = loc.AsFpuRegisters().fpu_reg();
    ASSERT_LESS_OR_EQUAL(dst_reg, CallbackContext::kNumDoubleArguments - 1);
    ctxt->double_arguments[dst_reg] = value;
  } else {
    FATAL("Unimplemented location %s", loc.ToCString(thread->zone()));
  }
}

static void PassFfiCallbackCompoundReturnPortion(
    Thread* thread,
    CallbackContext* ctxt,
    const compiler::ffi::NativeLocation& loc,
    const Instance& compound_contents,
    intptr_t offset_in_bytes) {
  auto const size = loc.payload_type().SizeInBytes();
  if (loc.IsMultiple()) {
    // Copy from the TypedData to from a list of native locations.
    const auto& multiple = loc.AsMultiple();
    for (intptr_t i = 0, n = multiple.locations().length(); i < n; ++i) {
      const auto& portion = *multiple.locations().At(i);
      PassFfiCallbackCompoundReturnPortion(thread, ctxt, portion,
                                           compound_contents, offset_in_bytes);
      offset_in_bytes += portion.payload_type().SizeInBytes();
    }
  } else {
    if (!loc.payload_type().IsPrimitive()) {
      FATAL("Unimplemented location %s", loc.ToCString(thread->zone()));
    }
    ASSERT(size <= kWordSize);
    uword value;
    {
      NoSafepointScope scope;
      auto* const src = GetDataAddress(compound_contents, offset_in_bytes);
      memcpy(&value, src, size);  // NOLINT
    }
    SetLocationInCallbackContext(thread, ctxt, loc, value);
  }
}

static void PassFfiCallbackResult(
    Thread* thread,
    CallbackContext* ctxt,
    const compiler::ffi::CallbackMarshaller& marshaller,
    const Object& result) {
  const intptr_t arg_index = compiler::ffi::kResultIndex;
  auto* const zone = thread->zone();
  const auto& loc = marshaller.Location(arg_index);
  PRINT_IF_TRACING_INTERPRETER("Return value -> %s\n", loc.ToCString());
  if (marshaller.IsCompoundCType(arg_index)) {
    auto& compound_contents = Instance::Handle(zone);
    intptr_t offset_in_bytes = 0;

    if (result.IsInstance()) {
      auto* const object_store = thread->isolate_group()->object_store();
      const auto& typed_data_field =
          Field::Handle(zone, object_store->compound_typed_data_base_field());
      const auto& offset_in_bytes_field =
          Field::Handle(zone, object_store->compound_offset_in_bytes_field());

      compound_contents ^= Instance::Cast(result).GetField(typed_data_field);
      offset_in_bytes = Smi::Value(
          Smi::RawCast(Instance::Cast(result).GetField(offset_in_bytes_field)));
    } else {
      ASSERT(result.IsError());
      PRINT_IF_TRACING_INTERPRETER("returning a zero-filled struct instead\n");
      // The exceptional return for a compound type is always zero-filled.
      const intptr_t container_size = loc.container_type().SizeInBytes();
      compound_contents =
          TypedData::New(kTypedDataInt8ArrayCid, container_size);
    }

    if (loc.IsPointerToMemory()) {
      // This is a compound value with contents that should be copied into
      // the space pointed to by the pointer location, instead of replacing
      // the pointer stored in the pointer location.
      const auto& ptr_loc = loc.AsPointerToMemory().pointer_location();
      const uword ptr = RetrieveLocationFromCallbackContext(thread, *ctxt,
                                                            arg_index, ptr_loc);
      auto* const dst = reinterpret_cast<void*>(ptr);
      NoSafepointScope scope;
      const void* const src =
          GetDataAddress(compound_contents, offset_in_bytes);
      memcpy(dst, src, loc.payload_type().SizeInBytes());  // NOLINT
    } else {
      PassFfiCallbackCompoundReturnPortion(thread, ctxt, loc, compound_contents,
                                           offset_in_bytes);
    }
  } else {
    if (!loc.payload_type().IsPrimitive()) {
      FATAL("Unimplemented location %s", loc.ToCString(zone));
    }
    ASSERT(loc.payload_type().SizeInBytes() <= kWordSize);
    // FfiConvertPrimitiveToNative handles the exceptional case for
    // IsVoid and IsPointerPointer.
    const uword value =
        FfiConvertPrimitiveToNative(thread, marshaller, arg_index, result);
    SetLocationInCallbackContext(thread, ctxt, loc, value);
  }
  PRINT_IF_TRACING_INTERPRETER(
      "%s", ctxt->ToCString(zone, marshaller.StackTopInBytes()));
}
#endif  // !defined(DART_PRECOMPILED_RUNTIME)

extern "C" void DLRT_DoInterpretedFfiCallback(
    Thread* thread,
    CallbackContext* ctxt,
    const PersistentHandle* function_handle) {
#if !defined(DART_PRECOMPILED_RUNTIME)
  ASSERT_EQUAL(thread->execution_state(), Thread::kThreadInVM);
  // Either this is a callback without an active zone (e.g., isolate
  // group bound callbacks) or avoid leaking handles in the caller's zone.
  StackZone stack_zone(thread);
  auto* const zone = stack_zone.GetZone();

  ASSERT(function_handle != nullptr);
  const auto& function =
      Function::ZoneHandle(zone, Function::RawCast(function_handle->ptr()));
  ASSERT(!function.IsNull());
  ASSERT(function.IsFfiCallbackTrampoline());
  const bool is_async =
      function.GetFfiCallbackKind() == FfiCallbackKind::kAsyncCallback;

  const int64_t context = thread->unboxed_int64_runtime_arg();
  PRINT_IF_TRACING_INTERPRETER("calling %s FFI callback, context %#" Px64 "\n",
                               is_async ? "async" : "sync", context);
  auto& closure = Closure::Handle(zone);
  if (!is_async) {
    // The context stores a pointer to a persistent handle, if any.
    if (auto* const handle = reinterpret_cast<PersistentHandle*>(context)) {
      closure = Closure::RawCast(handle->ptr());
    }
  }

  const char* error = nullptr;
  auto* const marshaller_ptr =
      compiler::ffi::CallbackMarshaller::FromFunction(zone, function, &error);
  // AbiSpecificTypes can have an incomplete mapping.
  if (error != nullptr) {
    const auto& language_error = Error::Handle(
        LanguageError::New(String::Handle(String::New(error, Heap::kOld)),
                           Report::kError, Heap::kOld));
    Report::LongJump(language_error);
  }
  RELEASE_ASSERT(marshaller_ptr != nullptr);
  const auto& marshaller = *marshaller_ptr;

  PRINT_IF_TRACING_INTERPRETER(
      "%s", ctxt->ToCString(zone, marshaller.StackTopInBytes()));

  const intptr_t type_args_len = 0;
  const intptr_t num_implicit_args = !closure.IsNull() ? 1 : 0;
  intptr_t num_args = num_implicit_args + marshaller.num_args();
  const auto& arguments = Array::Handle(zone, Array::New(num_args));
  const auto& argdesc = Array::Handle(
      zone, ArgumentsDescriptor::NewBoxed(type_args_len, num_args));

  // Convert all the native arguments to Dart values.
  if (!closure.IsNull()) {
    arguments.SetAt(0, closure);
  }
  auto& obj = Object::Handle(zone);
  for (intptr_t i = num_implicit_args; i < num_args; ++i) {
    obj = ReceiveFfiCallbackArgument(thread, *ctxt, marshaller,
                                     i - num_implicit_args);
    arguments.SetAt(i, obj);
  }

  PRINT_IF_TRACING_INTERPRETER("performing FFI callback %s\n",
                               function.ToFullyQualifiedCString());
  if (is_async) {
    PRINT_IF_TRACING_INTERPRETER(
        "sending arguments to listener of async FFI callback\n");
    // See DRT_FfiAsyncCallbackSend.
    Dart_Port target_port = static_cast<Dart_Port>(context);
    const Array& msg_array = Array::Handle(zone, Array::New(3));
    msg_array.SetAt(0, arguments);
    PersistentHandle* handle =
        thread->isolate_group()->api_state()->AllocatePersistentHandle();
    handle->set_ptr(msg_array);
    PortMap::PostMessage(
        Message::New(target_port, handle, Message::kNormalPriority));
  } else {
    auto& target = Function::Handle(zone, function.FfiCallbackTarget());
    if (target.IsNull()) {
      ASSERT(!closure.IsNull());
      target = closure.function();
    }
    ASSERT(!target.IsNull());
    ASSERT(target.IsInterpreted());
    PRINT_IF_TRACING_INTERPRETER("calling FFI callback target %s\n",
                                 target.ToFullyQualifiedCString());

    auto& result = Object::Handle(zone);
    {
      // Interpreter::Current() ensures the current thread has an interpreter,
      // which it may not if this is an isolate group bound callback with a
      // fresh isolate.
      ASSERT(thread == Thread::Current());
      auto* const interpreter = Interpreter::Current();
      TransitionVMToGenerated transition(thread);
      result = interpreter->Call(target, argdesc, arguments, thread);
    }
    if (result.IsError()) {
      PRINT_IF_TRACING_INTERPRETER("FFI callback threw exception %s\n",
                                   Error::Cast(result).ToErrorCString());
      // See catch body in FlowGraphBuilder::BuildGraphOfSyncFfiCallback.
      if (function.FfiCallbackExceptionalReturn() != Object::null()) {
        result = function.FfiCallbackExceptionalReturn();
        PRINT_IF_TRACING_INTERPRETER("returning %s instead\n",
                                     result.ToCString());
      } else if (marshaller.IsHandleCType(compiler::ffi::kResultIndex)) {
        PRINT_IF_TRACING_INTERPRETER("returning exception as handle\n");
      } else {
        ASSERT(marshaller.IsVoid(compiler::ffi::kResultIndex) ||
               marshaller.IsPointerPointer(compiler::ffi::kResultIndex) ||
               marshaller.IsCompoundCType(compiler::ffi::kResultIndex));
        // PassFfiCallbackResult prints an appropriate trace statement.
      }
    } else {
      PRINT_IF_TRACING_INTERPRETER("returning from FFI callback with %s\n",
                                   result.ToCString());
    }
    PassFfiCallbackResult(thread, ctxt, marshaller, result);
  }
  // The first-level trampoline calls the epilogue after this returns.
#else
  UNIMPLEMENTED();  // Interpreted callbacks are not currently handled in AOT.
#endif
}
#endif  // defined(HOST_ARCH_ARM64) && defined(DART_DYNAMIC_MODULES)

const char* CallbackContext::ToCString(Zone* zone,
                                       intptr_t stack_top_in_bytes,
                                       bool print_stack_as_words) const {
  ZoneTextBuffer buffer(zone);
  buffer.AddString("CallbackContext {\n");
  buffer.AddString("  Integer registers:\n");
  for (intptr_t i = 0; i < kNumIntegerArguments; ++i) {
    auto const reg = static_cast<Register>(i);
    buffer.Printf("    %5s: %#" Px "\n", RegisterNames::RegisterAbiName(reg),
                  integer_arguments[i]);
  }
  buffer.AddString("  Double registers:\n");
  for (intptr_t i = 0; i < kNumDoubleArguments; ++i) {
    auto const reg = static_cast<FpuRegister>(i);
    buffer.Printf("    %5s: %#" Px "\n", RegisterNames::FpuRegisterName(reg),
                  double_arguments[i]);
  }
  buffer.Printf("  Pointer to return struct (%s): %#" Px "\n",
                RegisterNames::RegisterAbiName(
                    CallingConventions::kPointerToReturnStructRegisterCall),
                return_struct_pointer);
  buffer.Printf("  Stack pointer: %#" Px "", sp);
  if (stack_top_in_bytes >= 0) {
    buffer.Printf(" (size %" Pd ")", stack_top_in_bytes);
    const intptr_t kIncrement = print_stack_as_words ? kWordSize : 1;
    const intptr_t kBytesPerLine = print_stack_as_words ? kWordSize : 16;
    for (intptr_t i = 0; i < stack_top_in_bytes; i += kIncrement) {
      if ((i % kBytesPerLine) == 0) {
        buffer.Printf("\n  %#" Px ": ", sp + i);
      } else if ((i % kWordSize == 0)) {
        buffer.AddString("  ");
      }
      if (print_stack_as_words) {
        buffer.Printf("%#" Px "", *reinterpret_cast<const uword*>(sp + i));
      } else {
        buffer.Printf("%.2x", *reinterpret_cast<const uint8_t*>(sp + i));
      }
    }
  }
  buffer.AddString("\n}\n");
  return buffer.buffer();
}

#if defined(HOST_ARCH_ARM64) && defined(SIMULATOR_FFI)
extern "C" void DoRedirectedFfiCallback(CallbackContext* ctxt,
                                        uword trampoline) {
  CallbackMetadata out;
  Thread* thread = DLRT_GetFfiCallbackMetadata(trampoline, &out);
  if (thread == nullptr) {
    // If GetFfiCallbackMetadata returned a null thread, it means that the async
    // callback was invoked after it was deleted. In this case, do nothing.
    return;
  }

  ASSERT(FLAG_use_simulator);
  Simulator* sim = Simulator::Current();
  ASSERT(sim != nullptr);
  sim->DoRedirectedFfiCallback(thread, ctxt, &out);
}
#endif  // defined(HOST_ARCH_ARM64) && defined(SIMULATOR_FFI)

// Check that argument types are valid for the given function.
// Arg0: function
// Arg1: arguments descriptor
// Arg2: arguments
// Return value: whether the arguments are valid
DEFINE_RUNTIME_ENTRY(CheckFunctionArgumentTypes, 3) {
#if defined(DART_DYNAMIC_MODULES)
  const auto& function = Function::CheckedHandle(zone, arguments.ArgAt(0));
  const auto& descriptor = Array::CheckedHandle(zone, arguments.ArgAt(1));
  const auto& args = Array::CheckedHandle(zone, arguments.ArgAt(2));

  const ArgumentsDescriptor args_desc(descriptor);
  if (function.AreValidArguments(args_desc, nullptr)) {
    const auto& result =
        Object::Handle(zone, function.DoArgumentTypesMatch(args, args_desc));
    if (result.IsError()) {
      Exceptions::PropagateError(Error::Cast(result));
    }
    arguments.SetReturn(Bool::True());
  } else {
    arguments.SetReturn(Bool::False());
  }
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES)
}

// Helper routine for tracing a type check.
static void PrintTypeCheck(const char* message,
                           const Instance& instance,
                           const AbstractType& type,
                           const TypeArguments& instantiator_type_arguments,
                           const TypeArguments& function_type_arguments,
                           const Bool& result) {
  DartFrameIterator iterator(Thread::Current(),
                             StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* caller_frame = iterator.NextFrame();
  ASSERT(caller_frame != nullptr);

  const AbstractType& instance_type =
      AbstractType::Handle(instance.GetType(Heap::kNew));
  ASSERT(instance_type.IsInstantiated() ||
         (instance.IsClosure() && instance_type.IsInstantiated(kCurrentClass)));
  LogBlock lb;
  if (type.IsInstantiated()) {
    THR_Print("%s: '%s' %d %s '%s' %d (pc: %#" Px ").\n", message,
              instance_type.NameCString(), instance_type.type_class_id(),
              (result.ptr() == Bool::True().ptr()) ? "is" : "is !",
              type.NameCString(), type.type_class_id(), caller_frame->pc());
  } else {
    // Instantiate type before printing.
    const AbstractType& instantiated_type = AbstractType::Handle(
        type.InstantiateFrom(instantiator_type_arguments,
                             function_type_arguments, kAllFree, Heap::kOld));
    THR_Print("%s: '%s' %s '%s' instantiated from '%s' (pc: %#" Px ").\n",
              message, instance_type.NameCString(),
              (result.ptr() == Bool::True().ptr()) ? "is" : "is !",
              instantiated_type.NameCString(), type.NameCString(),
              caller_frame->pc());
  }
  const Function& function =
      Function::Handle(caller_frame->LookupDartFunction());
  if (function.HasSavedArgumentsDescriptor()) {
    const auto& args_desc_array = Array::Handle(function.saved_args_desc());
    const ArgumentsDescriptor args_desc(args_desc_array);
    THR_Print(" -> Function %s [%s]\n", function.ToFullyQualifiedCString(),
              args_desc.ToCString());
  } else {
    THR_Print(" -> Function %s\n", function.ToFullyQualifiedCString());
  }
}

#if defined(TARGET_ARCH_IA32) || defined(DART_DYNAMIC_MODULES)
static BoolPtr CheckHashBasedSubtypeTestCache(
    Zone* zone,
    Thread* thread,
    const Instance& instance,
    const AbstractType& destination_type,
    const TypeArguments& instantiator_type_arguments,
    const TypeArguments& function_type_arguments,
    const SubtypeTestCache& cache) {
  ASSERT(cache.IsHash());
  // Record instances are not added to the cache as they don't have a valid
  // key (type of a record depends on types of all its fields).
  if (instance.IsRecord()) return Bool::null();
  Class& instance_class = Class::Handle(zone);
  if (instance.IsSmi()) {
    instance_class = Smi::Class();
  } else {
    instance_class = instance.clazz();
  }
  // If the type is uninstantiated and refers to parent function type
  // parameters, the function_type_arguments have been canonicalized
  // when concatenated.
  auto& instance_class_id_or_signature = Object::Handle(zone);
  auto& instance_type_arguments = TypeArguments::Handle(zone);
  auto& instance_parent_function_type_arguments = TypeArguments::Handle(zone);
  auto& instance_delayed_type_arguments = TypeArguments::Handle(zone);
  if (instance_class.IsClosureClass()) {
    const auto& closure = Closure::Cast(instance);
    const auto& function = Function::Handle(zone, closure.function());
    instance_class_id_or_signature = function.signature();
    instance_type_arguments = closure.instantiator_type_arguments();
    instance_parent_function_type_arguments = closure.function_type_arguments();
    instance_delayed_type_arguments = closure.delayed_type_arguments();
  } else {
    instance_class_id_or_signature = Smi::New(instance_class.id());
    if (instance_class.NumTypeArguments() > 0) {
      instance_type_arguments = instance.GetTypeArguments();
    }
  }

  intptr_t index = -1;
  auto& result = Bool::Handle(zone);
  if (cache.HasCheck(instance_class_id_or_signature, destination_type,
                     instance_type_arguments, instantiator_type_arguments,
                     function_type_arguments,
                     instance_parent_function_type_arguments,
                     instance_delayed_type_arguments, &index, &result)) {
    return result.ptr();
  }

  return Bool::null();
}
#endif  // defined(TARGET_ARCH_IA32) || defined(DART_DYNAMIC_MODULES)

// This updates the type test cache, an array containing 8 elements:
// - instance class (or function if the instance is a closure)
// - instance type arguments (null if the instance class is not generic)
// - instantiator type arguments (null if the type is instantiated)
// - function type arguments (null if the type is instantiated)
// - instance parent function type arguments (null if instance is not a closure)
// - instance delayed type arguments (null if instance is not a closure)
// - destination type (null if the type was known at compile time)
// - test result
// It can be applied to classes with type arguments in which case it contains
// just the result of the class subtype test, not including the evaluation of
// type arguments.
// This operation is currently very slow (lookup of code is not efficient yet).
static void UpdateTypeTestCache(
    Zone* zone,
    Thread* thread,
    const Instance& instance,
    const AbstractType& destination_type,
    const TypeArguments& instantiator_type_arguments,
    const TypeArguments& function_type_arguments,
    const Bool& result,
    const SubtypeTestCache& new_cache) {
  ASSERT(!new_cache.IsNull());
  ASSERT(destination_type.IsCanonical());
  ASSERT(instantiator_type_arguments.IsCanonical());
  ASSERT(function_type_arguments.IsCanonical());
  if (instance.IsRecord()) {
    // Do not add record instances to cache as they don't have a valid
    // key (type of a record depends on types of all its fields).
    if (FLAG_trace_type_checks) {
      THR_Print("Not updating subtype test cache for the record instance.\n");
    }
    return;
  }
  Class& instance_class = Class::Handle(zone);
  if (instance.IsSmi()) {
    instance_class = Smi::Class();
  } else {
    instance_class = instance.clazz();
  }
  // If the type is uninstantiated and refers to parent function type
  // parameters, the function_type_arguments have been canonicalized
  // when concatenated.
  auto& instance_class_id_or_signature = Object::Handle(zone);
  auto& instance_type_arguments = TypeArguments::Handle(zone);
  auto& instance_parent_function_type_arguments = TypeArguments::Handle(zone);
  auto& instance_delayed_type_arguments = TypeArguments::Handle(zone);
  if (instance_class.IsClosureClass()) {
    const auto& closure = Closure::Cast(instance);
    const auto& function = Function::Handle(zone, closure.function());
    instance_class_id_or_signature = function.signature();
    ASSERT(instance_class_id_or_signature.IsFunctionType());
    instance_type_arguments = closure.instantiator_type_arguments();
    instance_parent_function_type_arguments = closure.function_type_arguments();
    instance_delayed_type_arguments = closure.delayed_type_arguments();
    ASSERT(instance_class_id_or_signature.IsCanonical());
    ASSERT(instance_type_arguments.IsCanonical());
    ASSERT(instance_parent_function_type_arguments.IsCanonical());
    ASSERT(instance_delayed_type_arguments.IsCanonical());
  } else {
    instance_class_id_or_signature = Smi::New(instance_class.id());
    if (instance_class.NumTypeArguments() > 0) {
      instance_type_arguments = instance.GetTypeArguments();
      ASSERT(instance_type_arguments.IsCanonical());
    }
  }
  if (FLAG_trace_type_checks) {
    const auto& instance_class_name =
        String::Handle(zone, instance_class.Name());
    TextBuffer buffer(256);
    buffer.Printf("  Updating test cache %#" Px " with result %s for:\n",
                  static_cast<uword>(new_cache.ptr()), result.ToCString());
    if (instance.IsString()) {
      buffer.Printf("    instance: '%s'\n", instance.ToCString());
    } else {
      buffer.Printf("    instance: %s\n", instance.ToCString());
    }
    buffer.Printf("    class: %s (%" Pd ")\n", instance_class_name.ToCString(),
                  instance_class.id());
    buffer.Printf(
        "    raw entry: [ %#" Px ", %#" Px ", %#" Px ", %#" Px ", %#" Px
        ", %#" Px ", %#" Px ", %#" Px " ]\n",
        static_cast<uword>(instance_class_id_or_signature.ptr()),
        static_cast<uword>(instance_type_arguments.ptr()),
        static_cast<uword>(instantiator_type_arguments.ptr()),
        static_cast<uword>(function_type_arguments.ptr()),
        static_cast<uword>(instance_parent_function_type_arguments.ptr()),
        static_cast<uword>(instance_delayed_type_arguments.ptr()),
        static_cast<uword>(destination_type.ptr()),
        static_cast<uword>(result.ptr()));
    THR_Print("%s", buffer.buffer());
  }
  {
    SafepointMutexLocker ml(
        thread->isolate_group()->subtype_test_cache_mutex());
    const intptr_t len = new_cache.NumberOfChecks();
    if (len >= FLAG_max_subtype_cache_entries) {
      if (FLAG_trace_type_checks) {
        THR_Print("Not updating subtype test cache as its length reached %d\n",
                  FLAG_max_subtype_cache_entries);
      }
      return;
    }
    intptr_t colliding_index = -1;
    auto& old_result = Bool::Handle(zone);
    if (new_cache.HasCheck(
            instance_class_id_or_signature, destination_type,
            instance_type_arguments, instantiator_type_arguments,
            function_type_arguments, instance_parent_function_type_arguments,
            instance_delayed_type_arguments, &colliding_index, &old_result)) {
      if (FLAG_trace_type_checks) {
        TextBuffer buffer(256);
        buffer.Printf("  Collision for test cache %#" Px " at index %" Pd ":\n",
                      static_cast<uword>(new_cache.ptr()), colliding_index);
        buffer.Printf("    entry: ");
        new_cache.WriteEntryToBuffer(zone, &buffer, colliding_index, "      ");
        THR_Print("%s\n", buffer.buffer());
      }
      if (old_result.ptr() != result.ptr()) {
        FATAL("Existing subtype test cache entry has result %s, not %s",
              old_result.ToCString(), result.ToCString());
      }
      // Some other isolate might have updated the cache between entry was
      // found missing and now.
      return;
    }
    const intptr_t new_index = new_cache.AddCheck(
        instance_class_id_or_signature, destination_type,
        instance_type_arguments, instantiator_type_arguments,
        function_type_arguments, instance_parent_function_type_arguments,
        instance_delayed_type_arguments, result);
    if (FLAG_trace_type_checks) {
      TextBuffer buffer(256);
      buffer.Printf("  Added new entry to test cache %#" Px " at index %" Pd
                    ":\n",
                    static_cast<uword>(new_cache.ptr()), new_index);
      buffer.Printf("    new entry: ");
      new_cache.WriteEntryToBuffer(zone, &buffer, new_index, "      ");
      THR_Print("%s\n", buffer.buffer());
    }
  }
}

// Check that the given instance is an instance of the given type.
// Tested instance may be null, because a null test cannot always be inlined,
// e.g 'null is T' yields true if T = Null, but false if T = bool.
// Arg0: instance being checked.
// Arg1: type.
// Arg2: type arguments of the instantiator of the type.
// Arg3: type arguments of the function of the type.
// Arg4: SubtypeTestCache.
// Return value: true or false.
DEFINE_RUNTIME_ENTRY(Instanceof, 5) {
  const Instance& instance = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const AbstractType& type =
      AbstractType::CheckedHandle(zone, arguments.ArgAt(1));
  const TypeArguments& instantiator_type_arguments =
      TypeArguments::CheckedHandle(zone, arguments.ArgAt(2));
  const TypeArguments& function_type_arguments =
      TypeArguments::CheckedHandle(zone, arguments.ArgAt(3));
  const SubtypeTestCache& cache =
      SubtypeTestCache::CheckedHandle(zone, arguments.ArgAt(4));
  ASSERT(type.IsFinalized());
  ASSERT(!type.IsDynamicType());  // No need to check assignment.
  ASSERT(!cache.IsNull());
#if defined(TARGET_ARCH_IA32)
  // Hash-based caches are still not handled by the stubs on IA32.
  if (cache.IsHash()) {
    const auto& result = Bool::Handle(
        zone, CheckHashBasedSubtypeTestCache(zone, thread, instance, type,
                                             instantiator_type_arguments,
                                             function_type_arguments, cache));
    if (!result.IsNull()) {
      // Early exit because an entry already exists in the cache.
      arguments.SetReturn(result);
      return;
    }
  }
#endif  // defined(TARGET_ARCH_IA32)
  const Bool& result = Bool::Get(instance.IsInstanceOf(
      type, instantiator_type_arguments, function_type_arguments));
  if (FLAG_trace_type_checks) {
    PrintTypeCheck("InstanceOf", instance, type, instantiator_type_arguments,
                   function_type_arguments, result);
  }
  UpdateTypeTestCache(zone, thread, instance, type, instantiator_type_arguments,
                      function_type_arguments, result, cache);
  arguments.SetReturn(result);
}

#if defined(TESTING)
// Used only in type_testing_stubs_test.cc. If DRT_TypeCheck is entered, then
// this flag is set to true.
thread_local bool TESTING_runtime_entered_on_TTS_invocation = false;
#endif

// Check that the type of the given instance is a subtype of the given type and
// can therefore be assigned.
// Tested instance may not be null, because a null test is always inlined.
// Arg0: instance being assigned.
// Arg1: type being assigned to.
// Arg2: type arguments of the instantiator of the type being assigned to.
// Arg3: type arguments of the function of the type being assigned to.
// Arg4: name of variable being assigned to.
// Arg5: SubtypeTestCache.
// Arg6: invocation mode (see TypeCheckMode)
// Return value: instance if a subtype, otherwise throw a TypeError.
DEFINE_RUNTIME_ENTRY(TypeCheck, 7) {
  const Instance& src_instance =
      Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const AbstractType& dst_type =
      AbstractType::CheckedHandle(zone, arguments.ArgAt(1));
  const TypeArguments& instantiator_type_arguments =
      TypeArguments::CheckedHandle(zone, arguments.ArgAt(2));
  const TypeArguments& function_type_arguments =
      TypeArguments::CheckedHandle(zone, arguments.ArgAt(3));
  String& dst_name = String::Handle(zone);
  dst_name ^= arguments.ArgAt(4);
  ASSERT(dst_name.IsNull() || dst_name.IsString());

  SubtypeTestCache& cache = SubtypeTestCache::Handle(zone);
  cache ^= arguments.ArgAt(5);
  ASSERT(cache.IsNull() || cache.IsSubtypeTestCache());

  const TypeCheckMode mode = static_cast<TypeCheckMode>(
      Smi::CheckedHandle(zone, arguments.ArgAt(6)).Value());

#if defined(TESTING)
  TESTING_runtime_entered_on_TTS_invocation = true;
#endif

#if defined(TARGET_ARCH_IA32)
  ASSERT(mode == kTypeCheckFromInline);
#endif

#if defined(TARGET_ARCH_IA32) || defined(DART_DYNAMIC_MODULES)
  // Hash-based caches are not handled by the inline AssertAssignable
  // on IA32 and in the interpreter.
  if ((mode == kTypeCheckFromInline) && cache.IsHash()) {
    const auto& result = Bool::Handle(
        zone, CheckHashBasedSubtypeTestCache(
                  zone, thread, src_instance, dst_type,
                  instantiator_type_arguments, function_type_arguments, cache));
    if (result.ptr() == Bool::True().ptr()) {
      // Early exit because an entry already exists in the cache.
      arguments.SetReturn(src_instance);
      return;
    }
  }
#endif  // defined(TARGET_ARCH_IA32) || defined(DART_DYNAMIC_MODULES)

  // This is guaranteed on the calling side.
  ASSERT(!dst_type.IsDynamicType());

  const bool is_instance_of = src_instance.IsInstanceOf(
      dst_type, instantiator_type_arguments, function_type_arguments);

  if (FLAG_trace_type_checks) {
    PrintTypeCheck("TypeCheck", src_instance, dst_type,
                   instantiator_type_arguments, function_type_arguments,
                   Bool::Get(is_instance_of));
  }

  // Most paths through this runtime entry don't need to know what the
  // destination name was or if this was a dynamic assert assignable call,
  // so only walk the stack to find the stored destination name when necessary.
  auto resolve_dst_name = [&]() {
    if (!dst_name.IsNull()) return;
#if !defined(TARGET_ARCH_IA32)
    // Can only come here from type testing stub.
    ASSERT(mode != kTypeCheckFromInline);

    // Grab the [dst_name] from the pool.  It's stored at one pool slot after
    // the subtype-test-cache.
    DartFrameIterator iterator(thread,
                               StackFrameIterator::kNoCrossThreadIteration);
    StackFrame* caller_frame = iterator.NextFrame();
    const Code& caller_code =
        Code::Handle(zone, caller_frame->LookupDartCode());
    const ObjectPool& pool =
        ObjectPool::Handle(zone, caller_code.GetObjectPool());
    TypeTestingStubCallPattern tts_pattern(caller_frame->pc());
    const intptr_t stc_pool_idx = tts_pattern.GetSubtypeTestCachePoolIndex();
    const intptr_t dst_name_idx = stc_pool_idx + 1;
    dst_name ^= pool.ObjectAt(dst_name_idx);
#else
    UNREACHABLE();
#endif
  };

  if (!is_instance_of) {
    resolve_dst_name();
    if (dst_name.ptr() ==
        Symbols::dynamic_assert_assignable_stc_check().ptr()) {
#if !defined(TARGET_ARCH_IA32)
      // Can only come here from type testing stub via dynamic AssertAssignable.
      ASSERT(mode != kTypeCheckFromInline);
#endif
      // This was a dynamic closure call where the destination name was not
      // known at compile-time. Thus, fetch the original arguments and arguments
      // descriptor and re-do the type check  in the runtime, which causes the
      // error with the proper destination name to be thrown.
      DartFrameIterator iterator(thread,
                                 StackFrameIterator::kNoCrossThreadIteration);
      StackFrame* caller_frame = iterator.NextFrame();
      ASSERT(!caller_frame->is_interpreted());
      const auto& dispatcher =
          Function::Handle(zone, caller_frame->LookupDartFunction());
      ASSERT(dispatcher.IsInvokeFieldDispatcher());
      const auto& orig_arguments_desc =
          Array::Handle(zone, dispatcher.saved_args_desc());
      const ArgumentsDescriptor args_desc(orig_arguments_desc);
      const intptr_t arg_count = args_desc.CountWithTypeArgs();
      const auto& orig_arguments = Array::Handle(zone, Array::New(arg_count));
      auto& obj = Object::Handle(zone);
      for (intptr_t i = 0; i < arg_count; i++) {
        obj = *reinterpret_cast<ObjectPtr*>(
            ParamAddress(caller_frame->fp(), arg_count - i));
        orig_arguments.SetAt(i, obj);
      }
      const auto& receiver = Closure::CheckedHandle(
          zone, orig_arguments.At(args_desc.FirstArgIndex()));
      const auto& function = Function::Handle(zone, receiver.function());
      const auto& result = Object::Handle(
          zone, function.DoArgumentTypesMatch(orig_arguments, args_desc));
      if (result.IsError()) {
        Exceptions::PropagateError(Error::Cast(result));
      }
      // IsInstanceOf returned false, so we should have thrown a type
      // error in DoArgumentsTypesMatch.
      UNREACHABLE();
    }

    ASSERT(!dst_name.IsNull());
    // Throw a dynamic type error.
    const TokenPosition location = GetCallerLocation();
    const auto& src_type =
        AbstractType::Handle(zone, src_instance.GetType(Heap::kNew));
    auto& reported_type = AbstractType::Handle(zone, dst_type.ptr());
    if (!reported_type.IsInstantiated()) {
      // Instantiate dst_type before reporting the error.
      reported_type = reported_type.InstantiateFrom(instantiator_type_arguments,
                                                    function_type_arguments,
                                                    kAllFree, Heap::kNew);
    }
    Exceptions::CreateAndThrowTypeError(location, src_type, reported_type,
                                        dst_name);
    UNREACHABLE();
  }

  bool should_update_cache = true;
#if !defined(TARGET_ARCH_IA32)
  bool would_update_cache_if_not_lazy = false;
#if !defined(DART_PRECOMPILED_RUNTIME)
  // Checks against type parameters are done by loading the corresponding type
  // argument at runtime and calling the type argument's TTS. Thus, we install
  // specialized TTSes on the type argument, not the parameter itself.
  auto& tts_type = AbstractType::Handle(zone, dst_type.ptr());
  if (tts_type.IsTypeParameter()) {
    const auto& param = TypeParameter::Cast(tts_type);
    tts_type = param.GetFromTypeArguments(instantiator_type_arguments,
                                          function_type_arguments);
  }
  ASSERT(!tts_type.IsTypeParameter());

  if (mode == kTypeCheckFromLazySpecializeStub) {
    if (FLAG_trace_type_checks) {
      THR_Print("  Specializing type testing stub for %s\n",
                tts_type.ToCString());
    }
    const Code& code = Code::Handle(
        zone, TypeTestingStubGenerator::SpecializeStubFor(thread, tts_type));
    tts_type.SetTypeTestingStub(code);

    // Only create the cache if we failed to create a specialized TTS and doing
    // the same check would cause an update to the cache.
    would_update_cache_if_not_lazy =
        (!src_instance.IsNull() &&
         tts_type.type_test_stub() ==
             StubCode::DefaultNullableTypeTest().ptr()) ||
        tts_type.type_test_stub() == StubCode::DefaultTypeTest().ptr();
    should_update_cache = would_update_cache_if_not_lazy && cache.IsNull();
  }

  // Since dst_type is not a top type or type parameter, then the only default
  // stubs it can use are DefaultTypeTest or DefaultNullableTypeTest.
  if ((mode == kTypeCheckFromSlowStub) &&
      (tts_type.type_test_stub() != StubCode::DefaultNullableTypeTest().ptr() &&
       tts_type.type_test_stub() != StubCode::DefaultTypeTest().ptr())) {
    // The specialized type testing stub returned a false negative. That means
    // the specialization may have been generated using outdated cid ranges and
    // new classes appeared since the stub was generated. Try respecializing.
    if (FLAG_trace_type_checks) {
      THR_Print("  Rebuilding type testing stub for %s\n",
                tts_type.ToCString());
    }
    const auto& old_code = Code::Handle(zone, tts_type.type_test_stub());
    const auto& new_code = Code::Handle(
        zone, TypeTestingStubGenerator::SpecializeStubFor(thread, tts_type));
    ASSERT(old_code.ptr() != new_code.ptr());
    // A specialized stub should always respecialize to a non-default stub.
    ASSERT(new_code.ptr() != StubCode::DefaultNullableTypeTest().ptr() &&
           new_code.ptr() != StubCode::DefaultTypeTest().ptr());
    const auto& old_instructions =
        Instructions::Handle(old_code.instructions());
    const auto& new_instructions =
        Instructions::Handle(new_code.instructions());
    // Check if specialization produced exactly the same sequence of
    // instructions. If it did, then we have a false negative, which can
    // happen in some cases involving uninstantiated types. In these cases,
    // update the cache, because the only case in which these false negatives
    // could possibly turn into true positives is with reloads, which clear
    // all the SubtypeTestCaches.
    should_update_cache = old_instructions.Equals(new_instructions);
    if (FLAG_trace_type_checks) {
      THR_Print("  %s rebuilt type testing stub for %s\n",
                should_update_cache ? "Discarding" : "Installing",
                tts_type.ToCString());
    }
    if (!should_update_cache) {
      tts_type.SetTypeTestingStub(new_code);
    }
  }
#endif  // !defined(DART_PRECOMPILED_RUNTIME)
#endif  // !defined(TARGET_ARCH_IA32)

  if (should_update_cache) {
    if (cache.IsNull()) {
#if !defined(TARGET_ARCH_IA32)
      ASSERT(mode == kTypeCheckFromSlowStub ||
             (mode == kTypeCheckFromLazySpecializeStub &&
              would_update_cache_if_not_lazy));
      // We lazily create [SubtypeTestCache] for those call sites which actually
      // need one and will patch the pool entry.
      DartFrameIterator iterator(thread,
                                 StackFrameIterator::kNoCrossThreadIteration);
      StackFrame* caller_frame = iterator.NextFrame();
      ASSERT(!caller_frame->is_interpreted());
      const Code& caller_code =
          Code::Handle(zone, caller_frame->LookupDartCode());
      const ObjectPool& pool =
          ObjectPool::Handle(zone, caller_code.GetObjectPool());
      TypeTestingStubCallPattern tts_pattern(caller_frame->pc());
      const intptr_t stc_pool_idx = tts_pattern.GetSubtypeTestCachePoolIndex();
      // Ensure we do have a STC (lazily create it if not) and all threads use
      // the same STC.
      {
        SafepointMutexLocker ml(
            thread->isolate_group()->subtype_test_cache_mutex());
        cache ^= pool.ObjectAt<std::memory_order_acquire>(stc_pool_idx);
        if (cache.IsNull()) {
          resolve_dst_name();
          // If this is a dynamic AssertAssignable check, then we must assume
          // all inputs may be needed, as the type may vary from call to call.
          const intptr_t num_inputs =
              dst_name.ptr() ==
                      Symbols::dynamic_assert_assignable_stc_check().ptr()
                  ? SubtypeTestCache::kMaxInputs
                  : SubtypeTestCache::UsedInputsForType(dst_type);
          cache = SubtypeTestCache::New(num_inputs);
          pool.SetObjectAt<std::memory_order_release>(stc_pool_idx, cache);
          if (FLAG_trace_type_checks) {
            THR_Print("  Installed new subtype test cache %#" Px " with %" Pd
                      " inputs at index %" Pd " of pool for %s\n",
                      static_cast<uword>(cache.ptr()), num_inputs, stc_pool_idx,
                      caller_code.ToCString());
          }
        }
      }
#else
      UNREACHABLE();
#endif
    }

    UpdateTypeTestCache(zone, thread, src_instance, dst_type,
                        instantiator_type_arguments, function_type_arguments,
                        Bool::True(), cache);
  }

  arguments.SetReturn(src_instance);
}

DEFINE_RUNTIME_ENTRY(Throw, 1) {
  if (FLAG_gc_at_throw) {
    thread->isolate_group()->heap()->CollectGarbage(thread, GCType::kEvacuate,
                                                    GCReason::kDebugging);
    thread->isolate_group()->heap()->CollectAllGarbage(GCReason::kDebugging,
                                                       /*compact=*/true);
  }

  const Instance& exception = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  Exceptions::Throw(thread, exception);
}

DEFINE_RUNTIME_ENTRY(ReThrow, 3) {
  if (FLAG_gc_at_throw) {
    thread->isolate_group()->heap()->CollectGarbage(thread, GCType::kEvacuate,
                                                    GCReason::kDebugging);
    thread->isolate_group()->heap()->CollectAllGarbage(GCReason::kDebugging,
                                                       /*compact=*/true);
  }

  const Instance& exception = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const Instance& stacktrace =
      Instance::CheckedHandle(zone, arguments.ArgAt(1));
  const Smi& bypass_debugger = Smi::CheckedHandle(zone, arguments.ArgAt(2));
  Exceptions::ReThrow(thread, exception, stacktrace,
                      bypass_debugger.Value() != 0);
}

// Patches static call in optimized code with the target's entry point.
// Compiles target if necessary.
DEFINE_RUNTIME_ENTRY(PatchStaticCall, 0) {
#if !defined(DART_PRECOMPILED_RUNTIME)
  DartFrameIterator iterator(thread,
                             StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* caller_frame = iterator.NextFrame();
  ASSERT(caller_frame != nullptr);
  ASSERT(!caller_frame->is_interpreted());
  const Code& caller_code = Code::Handle(zone, caller_frame->LookupDartCode());
  ASSERT(!caller_code.IsNull());
  ASSERT(caller_code.is_optimized());
  const Function& target_function = Function::Handle(
      zone, caller_code.GetStaticCallTargetFunctionAt(caller_frame->pc()));
  const Code& target_code = Code::Handle(zone, target_function.EnsureHasCode());
  // Before patching verify that we are not repeatedly patching to the same
  // target.
  if (target_code.ptr() !=
      CodePatcher::GetStaticCallTargetAt(caller_frame->pc(), caller_code)) {
    GcSafepointOperationScope safepoint(thread);
    if (target_code.ptr() !=
        CodePatcher::GetStaticCallTargetAt(caller_frame->pc(), caller_code)) {
      CodePatcher::PatchStaticCallAt(caller_frame->pc(), caller_code,
                                     target_code);
      caller_code.SetStaticCallTargetCodeAt(caller_frame->pc(), target_code);
      if (FLAG_trace_patching) {
        THR_Print("PatchStaticCall: patching caller pc %#" Px
                  ""
                  " to '%s' new entry point %#" Px " (%s)\n",
                  caller_frame->pc(), target_function.ToFullyQualifiedCString(),
                  target_code.EntryPoint(),
                  target_code.is_optimized() ? "optimized" : "unoptimized");
      }
    }
  }
  arguments.SetReturn(target_code);
#else
  UNREACHABLE();
#endif
}

#if defined(PRODUCT) || defined(DART_PRECOMPILED_RUNTIME)
DEFINE_RUNTIME_ENTRY(BreakpointRuntimeHandler, 0) {
  UNREACHABLE();
  return;
}
#else
// Gets called from debug stub when code reaches a breakpoint
// set on a runtime stub call.
DEFINE_RUNTIME_ENTRY(BreakpointRuntimeHandler, 0) {
  DartFrameIterator iterator(thread,
                             StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* caller_frame = iterator.NextFrame();
  ASSERT(caller_frame != nullptr);
  Object& orig_value = Object::Handle(zone);
  if (!caller_frame->is_interpreted()) {
    orig_value = thread->isolate_group()->debugger()->GetPatchedStubAddress(
        caller_frame->pc());
  } else {
    orig_value = Smi::New(thread->isolate_group()->debugger()->GetPatchedOpcode(
        caller_frame->pc()));
  }
  const Error& error =
      Error::Handle(zone, isolate->debugger()->PauseBreakpoint());
  ThrowIfError(error);
  arguments.SetReturn(orig_value);
}
#endif

DEFINE_RUNTIME_ENTRY(SingleStepHandler, 0) {
#if defined(PRODUCT) || defined(DART_PRECOMPILED_RUNTIME)
  UNREACHABLE();
#else
  const Error& error =
      Error::Handle(zone, isolate->debugger()->PauseStepping());
  ThrowIfError(error);
#endif
}

DEFINE_RUNTIME_ENTRY(ResumptionBreakpointHandler, 0) {
#if defined(DART_DYNAMIC_MODULES) && !defined(PRODUCT)
  isolate->debugger()->ResumptionBreakpoint();
#else
  UNREACHABLE();
#endif
}

// An instance call of the form o.f(...) could not be resolved.  Check if
// there is a getter with the same name.  If so, invoke it.  If the value is
// a closure, invoke it with the given arguments.  If the value is a
// non-closure, attempt to invoke "call" on it.
static bool ResolveCallThroughGetter(const Class& receiver_class,
                                     const String& target_name,
                                     const String& demangled,
                                     const Array& arguments_descriptor,
                                     bool allow_add,
                                     Function* result) {
  const String& getter_name = String::Handle(Field::GetterName(demangled));
  const int kTypeArgsLen = 0;
  const int kNumArguments = 1;
  ArgumentsDescriptor args_desc(Array::Handle(
      ArgumentsDescriptor::NewBoxed(kTypeArgsLen, kNumArguments)));
  const Function& getter =
      Function::Handle(Resolver::ResolveDynamicForReceiverClass(
          receiver_class, getter_name, args_desc, allow_add));
  if (getter.IsNull() || getter.IsMethodExtractor()) {
    return false;
  }
  // We do this on the target_name, _not_ on the demangled name, so that
  // FlowGraphBuilder::BuildGraphOfInvokeFieldDispatcher can detect dynamic
  // calls from the dyn: tag on the name of the dispatcher.
  const String& dispatcher_name = Function::DropImplicitCallPrefix(target_name);
  const Function& target_function =
      Function::Handle(receiver_class.GetInvocationDispatcher(
          dispatcher_name, arguments_descriptor,
          UntaggedFunction::kInvokeFieldDispatcher,
          /*create_if_absent=*/allow_add, getter.is_dynamically_callable()));
  ASSERT(!allow_add || !target_function.IsNull());
  if (FLAG_trace_ic) {
    OS::PrintErr(
        "InvokeField IC miss: adding <%s> id:%" Pd " -> <%s>\n",
        receiver_class.ToCString(), receiver_class.id(),
        target_function.IsNull() ? "null" : target_function.ToCString());
  }
  *result = target_function.ptr();
  return true;
}

// Handle other invocations (implicit closures, noSuchMethod).
FunctionPtr InlineCacheMissHelper(const Class& receiver_class,
                                  const Array& args_descriptor,
                                  const String& target_name,
                                  bool allow_add) {
  // Create a demangled version of the target_name, if necessary, This is used
  // for the field getter in ResolveCallThroughGetter and as the target name
  // for the NoSuchMethod dispatcher (if needed).
  const String* demangled = &target_name;
  if (Function::IsDynamicInvocationForwarderName(target_name)) {
    demangled = &String::Handle(
        Function::DemangleDynamicInvocationForwarderName(target_name));
  }
  const bool is_getter = Field::IsGetterName(*demangled);
  const bool is_dyn_implicit_call =
      target_name.ptr() == Symbols::DynamicImplicitCall().ptr();
  Function& result = Function::Handle();
  if (is_getter || (is_dyn_implicit_call && !receiver_class.IsClosureClass()) ||
      !ResolveCallThroughGetter(receiver_class, target_name, *demangled,
                                args_descriptor, allow_add, &result)) {
    ArgumentsDescriptor desc(args_descriptor);
    const Function& target_function =
        Function::Handle(receiver_class.GetInvocationDispatcher(
            *demangled, args_descriptor,
            UntaggedFunction::kNoSuchMethodDispatcher,
            /*create_if_absent=*/allow_add,
            /* is_dynamically_callable = */ true));
    if (FLAG_trace_ic) {
      OS::PrintErr(
          "NoSuchMethod IC miss: adding <%s> id:%" Pd " -> <%s>\n",
          receiver_class.ToCString(), receiver_class.id(),
          target_function.IsNull() ? "null" : target_function.ToCString());
    }
    result = target_function.ptr();
  }
  // May be null if in the precompiled runtime, in which case dispatch will be
  // handled by NoSuchMethodFromCallStub.
  ASSERT(!allow_add || !result.IsNull());
  return result.ptr();
}

#if !defined(DART_PRECOMPILED_RUNTIME)
static void TrySwitchInstanceCall(Thread* thread,
                                  StackFrame* caller_frame,
                                  const Code& caller_code,
                                  const Function& caller_function,
                                  const ICData& ic_data,
                                  const Function& target_function) {
  ASSERT(!target_function.IsNull());
  auto zone = thread->zone();

  // Monomorphic/megamorphic calls only check the receiver CID.
  if (ic_data.NumArgsTested() != 1) return;

  ASSERT(ic_data.rebind_rule() == ICData::kInstance);

  // Monomorphic/megamorphic calls don't record exactness.
  if (ic_data.is_tracking_exactness()) return;

#if !defined(PRODUCT)
  // Monomorphic/megamorphic do not check the isolate's stepping flag.
  if (thread->isolate_group()->has_attempted_stepping()) return;
#endif

  // Monomorphic/megamorphic calls are only for unoptimized code.
  if (caller_frame->is_interpreted()) return;
  ASSERT(!caller_code.is_optimized());

  // Code is detached from its function. This will prevent us from resetting
  // the switchable call later because resets are function based and because
  // the ic_data_array belongs to the function instead of the code. This should
  // only happen because of reload, but it sometimes happens with KBC mixed mode
  // probably through a race between foreground and background compilation.
  if (caller_function.unoptimized_code() != caller_code.ptr()) {
    return;
  }
#if !defined(PRODUCT)
  // Skip functions that contain breakpoints or when debugger is in single
  // stepping mode.
  if (thread->isolate_group()->debugger()->IsDebugging(thread,
                                                       caller_function)) {
    return;
  }
#endif

  const intptr_t num_checks = ic_data.NumberOfChecks();

  // Monomorphic call.
  if (FLAG_unopt_monomorphic_calls && (num_checks == 1)) {
    // A call site in the monomorphic state does not load the arguments
    // descriptor, so do not allow transition to this state if the callee
    // needs it.
    if (target_function.PrologueNeedsArgumentsDescriptor()) {
      return;
    }

    const Array& data = Array::Handle(zone, ic_data.entries());
    const Code& target = Code::Handle(zone, target_function.EnsureHasCode());
    CodePatcher::PatchInstanceCallAt(caller_frame->pc(), caller_code, data,
                                     target);
    if (FLAG_trace_ic) {
      OS::PrintErr("Instance call at %" Px
                   " switching to monomorphic dispatch, %s\n",
                   caller_frame->pc(), ic_data.ToCString());
    }
    return;  // Success.
  }

  // Megamorphic call.
  if (FLAG_unopt_megamorphic_calls &&
      (num_checks > FLAG_max_polymorphic_checks)) {
    const String& name = String::Handle(zone, ic_data.target_name());
    const Array& descriptor =
        Array::Handle(zone, ic_data.arguments_descriptor());
    const MegamorphicCache& cache = MegamorphicCache::Handle(
        zone, MegamorphicCacheTable::Lookup(thread, name, descriptor));
    ic_data.set_is_megamorphic(true);
    CodePatcher::PatchInstanceCallAt(caller_frame->pc(), caller_code, cache,
                                     StubCode::MegamorphicCall());
    if (FLAG_trace_ic) {
      OS::PrintErr("Instance call at %" Px
                   " switching to megamorphic dispatch, %s\n",
                   caller_frame->pc(), ic_data.ToCString());
    }
    return;  // Success.
  }
}
#endif  // !defined(DART_PRECOMPILED_RUNTIME)

// Perform the subtype and return constant function based on the result.
static FunctionPtr ComputeTypeCheckTarget(const Instance& receiver,
                                          const AbstractType& type,
                                          const ArgumentsDescriptor& desc) {
  const bool result = receiver.IsInstanceOf(type, Object::null_type_arguments(),
                                            Object::null_type_arguments());
  const ObjectStore* store = IsolateGroup::Current()->object_store();
  const Function& target =
      Function::Handle(result ? store->simple_instance_of_true_function()
                              : store->simple_instance_of_false_function());
  ASSERT(!target.IsNull());
  return target.ptr();
}

static FunctionPtr Resolve(
    Thread* thread,
    Zone* zone,
    const GrowableArray<const Instance*>& caller_arguments,
    const Class& receiver_class,
    const String& name,
    const Array& descriptor,
    bool allow_add) {
  ASSERT(name.IsSymbol());
  auto& target_function = Function::Handle(zone);
  ArgumentsDescriptor args_desc(descriptor);

  if (receiver_class.EnsureIsFinalized(thread) == Error::null()) {
    target_function = Resolver::ResolveDynamicForReceiverClass(
        receiver_class, name, args_desc, allow_add);
  }
  if (caller_arguments.length() == 2 &&
      target_function.ptr() == thread->isolate_group()
                                   ->object_store()
                                   ->simple_instance_of_function()) {
    // Replace the target function with constant function.
    const AbstractType& type = AbstractType::Cast(*caller_arguments[1]);
    target_function =
        ComputeTypeCheckTarget(*caller_arguments[0], type, args_desc);
  }

  if (target_function.IsNull()) {
    target_function =
        InlineCacheMissHelper(receiver_class, descriptor, name, allow_add);
  }
  ASSERT(!allow_add || !target_function.IsNull());
  return target_function.ptr();
}

// Handles a static call in unoptimized code that has one argument type not
// seen before. Compile the target if necessary and update the ICData.
// Arg0: argument.
// Arg1: IC data object.
DEFINE_RUNTIME_ENTRY(StaticCallMissHandlerOneArg, 2) {
  const Instance& arg = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const ICData& ic_data = ICData::CheckedHandle(zone, arguments.ArgAt(1));
  // IC data for static call is prepopulated with the statically known target.
  ASSERT(!ic_data.NumberOfChecksIs(0));
  const Function& target = Function::Handle(zone, ic_data.GetTargetAt(0));
  target.EnsureHasCode();
  ASSERT(!target.IsNull() && target.HasCode());
  ic_data.EnsureHasReceiverCheck(arg.GetClassId(), target, 1);
  if (FLAG_trace_ic) {
    DartFrameIterator iterator(thread,
                               StackFrameIterator::kNoCrossThreadIteration);
    StackFrame* caller_frame = iterator.NextFrame();
    ASSERT(caller_frame != nullptr);
    OS::PrintErr("StaticCallMissHandler at %#" Px " target %s (%" Pd ")\n",
                 caller_frame->pc(), target.ToCString(), arg.GetClassId());
  }
  arguments.SetReturn(target);
}

// Handles a static call in unoptimized code that has two argument types not
// seen before. Compile the target if necessary and update the ICData.
// Arg0: argument 0.
// Arg1: argument 1.
// Arg2: IC data object.
DEFINE_RUNTIME_ENTRY(StaticCallMissHandlerTwoArgs, 3) {
  const Instance& arg0 = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const Instance& arg1 = Instance::CheckedHandle(zone, arguments.ArgAt(1));
  const ICData& ic_data = ICData::CheckedHandle(zone, arguments.ArgAt(2));
  // IC data for static call is prepopulated with the statically known target.
  ASSERT(!ic_data.NumberOfChecksIs(0));
  const Function& target = Function::Handle(zone, ic_data.GetTargetAt(0));
  target.EnsureHasCode();
  GrowableArray<intptr_t> cids(2);
  cids.Add(arg0.GetClassId());
  cids.Add(arg1.GetClassId());
  ic_data.EnsureHasCheck(cids, target);
  if (FLAG_trace_ic) {
    DartFrameIterator iterator(thread,
                               StackFrameIterator::kNoCrossThreadIteration);
    StackFrame* caller_frame = iterator.NextFrame();
    ASSERT(caller_frame != nullptr);
    OS::PrintErr("StaticCallMissHandler at %#" Px " target %s (%" Pd ", %" Pd
                 ")\n",
                 caller_frame->pc(), target.ToCString(), cids[0], cids[1]);
  }
  arguments.SetReturn(target);
}

#if defined(DART_PRECOMPILED_RUNTIME)

static bool IsSingleTarget(IsolateGroup* isolate_group,
                           Zone* zone,
                           intptr_t lower_cid,
                           intptr_t upper_cid,
                           const Function& target,
                           const String& name) {
  Class& cls = Class::Handle(zone);
  ClassTable* table = isolate_group->class_table();
  Function& other_target = Function::Handle(zone);
  for (intptr_t cid = lower_cid; cid <= upper_cid; cid++) {
    if (!table->HasValidClassAt(cid)) continue;
    cls = table->At(cid);
    if (cls.is_abstract()) continue;
    if (!cls.is_allocated() && !cls.is_declared_in_bytecode()) continue;
    other_target = Resolver::ResolveDynamicAnyArgs(zone, cls, name,
                                                   /*allow_add=*/false);
    if (other_target.ptr() != target.ptr()) {
      return false;
    }
  }
  return true;
}

class SavedUnlinkedCallMapKeyEqualsTraits : public AllStatic {
 public:
  static const char* Name() { return "SavedUnlinkedCallMapKeyEqualsTraits "; }
  static bool ReportStats() { return false; }

  static bool IsMatch(const Object& key1, const Object& key2) {
    if (!key1.IsInteger() || !key2.IsInteger()) return false;
    return Integer::Cast(key1).Equals(Integer::Cast(key2));
  }
  static uword Hash(const Object& key) {
    return Integer::Cast(key).CanonicalizeHash();
  }
};

using UnlinkedCallMap = UnorderedHashMap<SavedUnlinkedCallMapKeyEqualsTraits>;

static void SaveUnlinkedCall(Zone* zone,
                             IsolateGroup* isolate_group,
                             uword frame_pc,
                             const UnlinkedCall& unlinked_call) {
  SafepointMutexLocker ml(isolate_group->unlinked_call_map_mutex());
  if (isolate_group->object_store()->saved_unlinked_calls() == Array::null()) {
    const auto& initial_map =
        Array::Handle(zone, HashTables::New<UnlinkedCallMap>(16, Heap::kOld));
    isolate_group->object_store()->set_saved_unlinked_calls(initial_map);
  }

  UnlinkedCallMap unlinked_call_map(
      zone, isolate_group->object_store()->saved_unlinked_calls());
  const auto& pc = Integer::Handle(zone, Integer::NewFromUint64(frame_pc));
  // Some other isolate might have updated unlinked_call_map[pc] too, but
  // their update should be identical to ours.
  const auto& new_or_old_value = UnlinkedCall::Handle(
      zone, UnlinkedCall::RawCast(
                unlinked_call_map.InsertOrGetValue(pc, unlinked_call)));
  RELEASE_ASSERT(new_or_old_value.ptr() == unlinked_call.ptr());
  isolate_group->object_store()->set_saved_unlinked_calls(
      unlinked_call_map.Release());
}

static UnlinkedCallPtr LoadUnlinkedCall(Zone* zone,
                                        IsolateGroup* isolate_group,
                                        uword pc) {
  SafepointMutexLocker ml(isolate_group->unlinked_call_map_mutex());
  ASSERT(isolate_group->object_store()->saved_unlinked_calls() !=
         Array::null());
  UnlinkedCallMap unlinked_call_map(
      zone, isolate_group->object_store()->saved_unlinked_calls());

  const auto& pc_integer = Integer::Handle(zone, Integer::NewFromUint64(pc));
  const auto& unlinked_call = UnlinkedCall::Cast(
      Object::Handle(zone, unlinked_call_map.GetOrDie(pc_integer)));
  isolate_group->object_store()->set_saved_unlinked_calls(
      unlinked_call_map.Release());
  return unlinked_call.ptr();
}

// NOTE: Right now we never delete [UnlinkedCall] objects. They are needed while
// a call site is in Unlinked/Monomorphic/MonomorphicSmiable/SingleTarget
// states.
//
// Theoretically we could free the [UnlinkedCall] object once we transition the
// call site to use ICData/MegamorphicCache, but that would require careful
// coordination between the deleter and a possible concurrent reader.
//
// To simplify the code we decided not to do that atm (only a very small
// fraction of callsites in AOT use switchable calls, the name/args-descriptor
// objects are kept alive anyways -> there is little memory savings from
// freeing the [UnlinkedCall] objects).

#endif  // defined(DART_PRECOMPILED_RUNTIME)

enum class MissHandler {
  kInlineCacheMiss,
  kSwitchableCallMiss,
  kFixCallersTargetMonomorphic,
};

// Handles updating of type feedback and possible patching of instance calls.
//
// It works in 3 separate steps:
//   - resolve the actual target
//   - update type feedback & (optionally) perform call site transition
//   - return the right values
//
// Depending on the JIT/AOT mode we obtain current and patch new (target, data)
// differently:
//
//   - JIT calls must be patched with CodePatcher::PatchInstanceCallAt()
//   - AOT calls must be patched with CodePatcher::PatchSwitchableCallAt()
//
// Independent of which miss handler was used or how we will return, we look at
// current (target, data) and see if we need to transition the call site to a
// new (target, data). We do this while holding `IG->patchable_call_mutex()`.
//
// Depending on which miss handler got called we might need to return
// differently:
//
//   - SwitchableCallMiss will get get (stub, data) return value
//   - InlineCache*Miss will get get function as return value
//
class PatchableCallHandler {
 public:
  PatchableCallHandler(Thread* thread,
                       const GrowableArray<const Instance*>& caller_arguments,
                       MissHandler miss_handler,
                       NativeArguments arguments,
                       StackFrame* caller_frame,
                       const Code& caller_code,
                       const Function& caller_function)
      : isolate_group_(thread->isolate_group()),
        thread_(thread),
        zone_(thread->zone()),
        caller_arguments_(caller_arguments),
        miss_handler_(miss_handler),
        arguments_(arguments),
        caller_frame_(caller_frame),
        caller_code_(caller_code),
        caller_function_(caller_function),
        name_(String::Handle()),
        args_descriptor_(Array::Handle()) {
    // We only have two arg IC calls in JIT mode.
    ASSERT(caller_arguments_.length() == 1 || !FLAG_precompiled_mode);
  }

  void ResolveSwitchAndReturn(const Object& data);

 private:
  FunctionPtr ResolveTargetFunction(const Object& data);

#if defined(DART_PRECOMPILED_RUNTIME)
  void HandleMissAOT(const Object& old_data,
                     uword old_entry,
                     const Function& target_function);

  void DoUnlinkedCallAOT(const UnlinkedCall& unlinked,
                         const Function& target_function);
  void DoMonomorphicMissAOT(const Object& old_data,
                            const Function& target_function);
  void DoSingleTargetMissAOT(const SingleTargetCache& data,
                             const Function& target_function);
  void DoICDataMissAOT(const ICData& data, const Function& target_function);
  bool CanExtendSingleTargetRange(const String& name,
                                  const Function& old_target,
                                  const Function& target_function,
                                  intptr_t* lower,
                                  intptr_t* upper);
#else
  void HandleMissJIT(const Object& old_data,
                     const Code& old_target,
                     const Function& target_function);

  void DoMonomorphicMissJIT(const Object& old_data,
                            const Function& target_function);
  void DoICDataMissJIT(const ICData& data,
                       const Object& old_data,
                       const Function& target_function);
#endif  // !defined(DART_PRECOMPILED_RUNTIME)
  void DoMegamorphicMiss(const MegamorphicCache& data,
                         const Function& target_function);

  void UpdateICDataWithTarget(const ICData& ic_data,
                              const Function& target_function);
  void TrySwitch(const ICData& ic_data, const Function& target_function);

  void ReturnAOT(const Code& stub, const Object& data);
  void ReturnJIT(const Code& stub, const Object& data, const Function& target);
  void ReturnJITorAOT(const Code& stub,
                      const Object& data,
                      const Function& target);

  const Instance& receiver() { return *caller_arguments_[0]; }

  bool should_consider_patching() {
    // In AOT we use switchable calls.
    if (FLAG_precompiled_mode) return true;

    // In JIT instance calls use a different calling sequence in unoptimized vs
    // optimized code (see [FlowGraphCompiler::EmitInstanceCallJIT] vs
    // [FlowGraphCompiler::EmitOptimizedInstanceCall]).
    //
    // The [CodePatcher::GetInstanceCallAt], [CodePatcher::PatchInstanceCallAt]
    // only recognize unoptimized call pattern.
    //
    // So we will not try to switch optimized instance calls.
    return !caller_code_.is_optimized();
  }

  ICDataPtr NewICData();
  ICDataPtr NewICDataWithTarget(intptr_t cid, const Function& target);

  IsolateGroup* isolate_group_;
  Thread* thread_;
  Zone* zone_;
  const GrowableArray<const Instance*>& caller_arguments_;
  MissHandler miss_handler_;
  NativeArguments arguments_;
  StackFrame* caller_frame_;
  const Code& caller_code_;
  const Function& caller_function_;

  // Call-site information populated during resolution.
  String& name_;
  Array& args_descriptor_;
  bool is_monomorphic_hit_ = false;
};

#if defined(DART_PRECOMPILED_RUNTIME)
void PatchableCallHandler::DoUnlinkedCallAOT(const UnlinkedCall& unlinked,
                                             const Function& target_function) {
  const auto& ic_data = ICData::Handle(
      zone_,
      target_function.IsNull()
          ? NewICData()
          : NewICDataWithTarget(receiver().GetClassId(), target_function));

  Object& object = Object::Handle(zone_, ic_data.ptr());
  Code& code = Code::Handle(zone_, StubCode::ICCallThroughCode().ptr());
  // If the target function has optional parameters or is generic, it's
  // prologue requires ARGS_DESC_REG to be populated. Yet the switchable calls
  // do not populate that on the call site, which is why we don't transition
  // those call sites to monomorphic, but rather directly to call via stub
  // (which will populate the ARGS_DESC_REG from the ICData).
  //
  // Because of this we also don't generate monomorphic checks for those
  // functions.
  if (!target_function.IsNull() &&
      !target_function.PrologueNeedsArgumentsDescriptor()) {
    // Patch to monomorphic call.
    ASSERT(target_function.HasCode());
    const Code& target_code =
        Code::Handle(zone_, target_function.CurrentCode());
    const Smi& expected_cid =
        Smi::Handle(zone_, Smi::New(receiver().GetClassId()));

    if (unlinked.can_patch_to_monomorphic()) {
      object = expected_cid.ptr();
      code = target_code.ptr();
      ASSERT(code.HasMonomorphicEntry());
    } else {
      object = MonomorphicSmiableCall::New(expected_cid.Value(), target_code);
      code = StubCode::MonomorphicSmiableCheck().ptr();
    }
  }
  CodePatcher::PatchSwitchableCallAt(caller_frame_->pc(), caller_code_, object,
                                     code);

  // Return the ICData. The miss stub will jump to continue in the IC lookup
  // stub.
  ReturnAOT(StubCode::ICCallThroughCode(), ic_data);
}

bool PatchableCallHandler::CanExtendSingleTargetRange(
    const String& name,
    const Function& old_target,
    const Function& target_function,
    intptr_t* lower,
    intptr_t* upper) {
  if (old_target.ptr() != target_function.ptr()) {
    return false;
  }
  intptr_t unchecked_lower, unchecked_upper;
  if (receiver().GetClassId() < *lower) {
    unchecked_lower = receiver().GetClassId();
    unchecked_upper = *lower - 1;
    *lower = receiver().GetClassId();
  } else {
    unchecked_upper = receiver().GetClassId();
    unchecked_lower = *upper + 1;
    *upper = receiver().GetClassId();
  }

  return IsSingleTarget(isolate_group_, zone_, unchecked_lower, unchecked_upper,
                        target_function, name);
}
#endif  // defined(DART_PRECOMPILED_RUNTIME)

#if defined(DART_PRECOMPILED_RUNTIME)
void PatchableCallHandler::DoMonomorphicMissAOT(
    const Object& old_data,
    const Function& target_function) {
  classid_t old_expected_cid;
  if (old_data.IsSmi()) {
    old_expected_cid = Smi::Cast(old_data).Value();
  } else {
    RELEASE_ASSERT(old_data.IsMonomorphicSmiableCall());
    old_expected_cid = MonomorphicSmiableCall::Cast(old_data).expected_cid();
  }
  const bool is_monomorphic_hit = old_expected_cid == receiver().GetClassId();
  const auto& old_receiver_class =
      Class::Handle(zone_, isolate_group_->class_table()->At(old_expected_cid));
  const auto& old_target = Function::Handle(
      zone_, Resolve(thread_, zone_, caller_arguments_, old_receiver_class,
                     name_, args_descriptor_, /*allow_add=*/false));

  const auto& ic_data = ICData::Handle(
      zone_, old_target.IsNull()
                 ? NewICData()
                 : NewICDataWithTarget(old_expected_cid, old_target));

  if (is_monomorphic_hit) {
    // The site just have been updated to monomorphic state with same
    // exact class id - do nothing in that case: stub will call through ic data.
    ReturnAOT(StubCode::ICCallThroughCode(), ic_data);
    return;
  }

  intptr_t lower = old_expected_cid;
  intptr_t upper = old_expected_cid;
  if (CanExtendSingleTargetRange(name_, old_target, target_function, &lower,
                                 &upper)) {
    const SingleTargetCache& cache =
        SingleTargetCache::Handle(zone_, SingleTargetCache::New());
    const Code& code = Code::Handle(zone_, target_function.CurrentCode());
    cache.set_target(code);
    cache.set_entry_point(code.EntryPoint());
    cache.set_lower_limit(lower);
    cache.set_upper_limit(upper);
    const Code& stub = StubCode::SingleTargetCall();
    CodePatcher::PatchSwitchableCallAt(caller_frame_->pc(), caller_code_, cache,
                                       stub);
    // Return the ICData. The miss stub will jump to continue in the IC call
    // stub.
    ReturnAOT(StubCode::ICCallThroughCode(), ic_data);
    return;
  }

  // Patch to call through stub.
  const Code& stub = StubCode::ICCallThroughCode();
  CodePatcher::PatchSwitchableCallAt(caller_frame_->pc(), caller_code_, ic_data,
                                     stub);

  // Return the ICData. The miss stub will jump to continue in the IC lookup
  // stub.
  ReturnAOT(stub, ic_data);
}
#endif  // defined(DART_PRECOMPILED_RUNTIME)

#if !defined(DART_PRECOMPILED_RUNTIME)
void PatchableCallHandler::DoMonomorphicMissJIT(
    const Object& old_data,
    const Function& target_function) {
  // Monomorphic calls use the ICData::entries() as their data.
  const auto& old_ic_data_entries = Array::Cast(old_data);
  // Any non-empty ICData::entries() has a backref to it's ICData.
  const auto& ic_data =
      ICData::Handle(zone_, ICData::ICDataOfEntriesArray(old_ic_data_entries));

  // The target didn't change, so we can stay inside monomorphic state.
  if (ic_data.NumberOfChecksIs(1) &&
      (ic_data.GetReceiverClassIdAt(0) == receiver().GetClassId())) {
    // No need to update ICData - it's already up-to-date.

    if (FLAG_trace_ic) {
      OS::PrintErr("Instance call at %" Px
                   " updating code (old code was disabled)\n",
                   caller_frame_->pc());
    }

    // We stay in monomorphic state, patch the code object and reload the icdata
    // entries array.
    const auto& code = Code::Handle(zone_, target_function.EnsureHasCode());
    const auto& data = Object::Handle(zone_, ic_data.entries());
    CodePatcher::PatchInstanceCallAt(caller_frame_->pc(), caller_code_, data,
                                     code);
    ReturnJIT(code, data, target_function);
    return;
  }

  ASSERT(ic_data.NumArgsTested() == 1);
  const Code& stub = ic_data.is_tracking_exactness()
                         ? StubCode::OneArgCheckInlineCacheWithExactnessCheck()
                         : StubCode::OneArgCheckInlineCache();
  if (FLAG_trace_ic) {
    OS::PrintErr("Instance call at %" Px
                 " switching monomorphic to polymorphic dispatch, %s\n",
                 caller_frame_->pc(), ic_data.ToCString());
  }
  CodePatcher::PatchInstanceCallAt(caller_frame_->pc(), caller_code_, ic_data,
                                   stub);

  ASSERT(caller_arguments_.length() == 1);
  UpdateICDataWithTarget(ic_data, target_function);
  ASSERT(should_consider_patching());
  TrySwitchInstanceCall(thread_, caller_frame_, caller_code_, caller_function_,
                        ic_data, target_function);
  ReturnJIT(stub, ic_data, target_function);
}
#endif  // !defined(DART_PRECOMPILED_RUNTIME)

#if defined(DART_PRECOMPILED_RUNTIME)
void PatchableCallHandler::DoSingleTargetMissAOT(
    const SingleTargetCache& data,
    const Function& target_function) {
  const Code& old_target_code = Code::Handle(zone_, data.target());
  const Function& old_target =
      Function::Handle(zone_, Function::RawCast(old_target_code.owner()));

  // We lost the original ICData when we patched to the monomorphic case.
  const auto& ic_data = ICData::Handle(
      zone_,
      target_function.IsNull()
          ? NewICData()
          : NewICDataWithTarget(receiver().GetClassId(), target_function));

  intptr_t lower = data.lower_limit();
  intptr_t upper = data.upper_limit();
  if (CanExtendSingleTargetRange(name_, old_target, target_function, &lower,
                                 &upper)) {
    data.set_lower_limit(lower);
    data.set_upper_limit(upper);
    // Return the ICData. The single target stub will jump to continue in the
    // IC call stub.
    ReturnAOT(StubCode::ICCallThroughCode(), ic_data);
    return;
  }

  // Call site is not single target, switch to call using ICData.
  const Code& stub = StubCode::ICCallThroughCode();
  CodePatcher::PatchSwitchableCallAt(caller_frame_->pc(), caller_code_, ic_data,
                                     stub);

  // Return the ICData. The single target stub will jump to continue in the
  // IC call stub.
  ReturnAOT(stub, ic_data);
}
#endif  // defined(DART_PRECOMPILED_RUNTIME)

#if defined(DART_PRECOMPILED_RUNTIME)
void PatchableCallHandler::DoICDataMissAOT(const ICData& ic_data,
                                           const Function& target_function) {
  const String& name = String::Handle(zone_, ic_data.target_name());
  const Class& cls = Class::Handle(zone_, receiver().clazz());
  ASSERT(!cls.IsNull());
  const Array& descriptor =
      Array::CheckedHandle(zone_, ic_data.arguments_descriptor());
  ArgumentsDescriptor args_desc(descriptor);
  if (FLAG_trace_ic || FLAG_trace_ic_miss_in_optimized) {
    OS::PrintErr("ICData miss, class=%s, function<%" Pd ">=%s\n",
                 cls.ToCString(), args_desc.TypeArgsLen(), name.ToCString());
  }

  if (target_function.IsNull()) {
    ReturnAOT(StubCode::NoSuchMethodDispatcher(), ic_data);
    return;
  }

  const intptr_t number_of_checks = ic_data.NumberOfChecks();

  if ((number_of_checks == 0) &&
      (!FLAG_precompiled_mode || ic_data.receiver_cannot_be_smi()) &&
      !target_function.PrologueNeedsArgumentsDescriptor()) {
    // This call site is unlinked: transition to a monomorphic direct call.
    // Note we cannot do this if the target has optional parameters because
    // the monomorphic direct call does not load the arguments descriptor.
    // We cannot do this if we are still in the middle of precompiling because
    // the monomorphic case hides a live instance selector from the
    // treeshaker.
    const Code& target_code =
        Code::Handle(zone_, target_function.EnsureHasCode());
    const Smi& expected_cid =
        Smi::Handle(zone_, Smi::New(receiver().GetClassId()));
    ASSERT(target_code.HasMonomorphicEntry());
    CodePatcher::PatchSwitchableCallAt(caller_frame_->pc(), caller_code_,
                                       expected_cid, target_code);
    ReturnAOT(target_code, expected_cid);
  } else {
    ic_data.EnsureHasReceiverCheck(receiver().GetClassId(), target_function);
    if (number_of_checks > FLAG_max_polymorphic_checks) {
      // Switch to megamorphic call.
      const MegamorphicCache& cache = MegamorphicCache::Handle(
          zone_, MegamorphicCacheTable::Lookup(thread_, name, descriptor));
      const Code& stub = StubCode::MegamorphicCall();

      CodePatcher::PatchSwitchableCallAt(caller_frame_->pc(), caller_code_,
                                         cache, stub);
      ReturnAOT(stub, cache);
    } else {
      ReturnAOT(StubCode::ICCallThroughCode(), ic_data);
    }
  }
}
#endif  // defined(DART_PRECOMPILED_RUNTIME)

#if !defined(DART_PRECOMPILED_RUNTIME)
void PatchableCallHandler::DoICDataMissJIT(const ICData& ic_data,
                                           const Object& old_code,
                                           const Function& target_function) {
  ASSERT(ic_data.NumArgsTested() == caller_arguments_.length());

  if (ic_data.NumArgsTested() == 1) {
    ASSERT(old_code.ptr() == StubCode::OneArgCheckInlineCache().ptr() ||
           old_code.ptr() ==
               StubCode::OneArgCheckInlineCacheWithExactnessCheck().ptr() ||
           old_code.ptr() ==
               StubCode::OneArgOptimizedCheckInlineCache().ptr() ||
           old_code.ptr() ==
               StubCode::OneArgOptimizedCheckInlineCacheWithExactnessCheck()
                   .ptr() ||
           old_code.ptr() == StubCode::ICCallBreakpoint().ptr() ||
           (old_code.IsNull() && !should_consider_patching()));
    UpdateICDataWithTarget(ic_data, target_function);
    if (should_consider_patching()) {
      TrySwitchInstanceCall(thread_, caller_frame_, caller_code_,
                            caller_function_, ic_data, target_function);
    }
    const Code& stub = Code::Handle(
        zone_, ic_data.is_tracking_exactness()
                   ? StubCode::OneArgCheckInlineCacheWithExactnessCheck().ptr()
                   : StubCode::OneArgCheckInlineCache().ptr());
    ReturnJIT(stub, ic_data, target_function);
  } else {
    ASSERT(old_code.ptr() == StubCode::TwoArgsCheckInlineCache().ptr() ||
           old_code.ptr() == StubCode::SmiAddInlineCache().ptr() ||
           old_code.ptr() == StubCode::SmiLessInlineCache().ptr() ||
           old_code.ptr() == StubCode::SmiEqualInlineCache().ptr() ||
           old_code.ptr() ==
               StubCode::TwoArgsOptimizedCheckInlineCache().ptr() ||
           old_code.ptr() == StubCode::ICCallBreakpoint().ptr() ||
           (old_code.IsNull() && !should_consider_patching()));
    UpdateICDataWithTarget(ic_data, target_function);
    ReturnJIT(StubCode::TwoArgsCheckInlineCache(), ic_data, target_function);
  }
}
#endif  // !defined(DART_PRECOMPILED_RUNTIME)

void PatchableCallHandler::DoMegamorphicMiss(const MegamorphicCache& data,
                                             const Function& target_function) {
  const String& name = String::Handle(zone_, data.target_name());
  const Class& cls = Class::Handle(zone_, receiver().clazz());
  ASSERT(!cls.IsNull());
  const Array& descriptor =
      Array::CheckedHandle(zone_, data.arguments_descriptor());
  ArgumentsDescriptor args_desc(descriptor);
  if (FLAG_trace_ic || FLAG_trace_ic_miss_in_optimized) {
    OS::PrintErr("Megamorphic miss, class=%s, function<%" Pd ">=%s\n",
                 cls.ToCString(), args_desc.TypeArgsLen(), name.ToCString());
  }
  if (target_function.IsNull()) {
    ReturnJITorAOT(StubCode::NoSuchMethodDispatcher(), data, target_function);
    return;
  }

  // Insert function found into cache.
  const Smi& class_id = Smi::Handle(zone_, Smi::New(cls.id()));
  data.EnsureContains(class_id, target_function);
  ReturnJITorAOT(StubCode::MegamorphicCall(), data, target_function);
}

void PatchableCallHandler::UpdateICDataWithTarget(
    const ICData& ic_data,
    const Function& target_function) {
  if (target_function.IsNull()) return;

  // If, upon return of the runtime, we will invoke the target directly we have
  // to increment the call count here in the ICData.
  // If we instead only insert a new ICData entry and will return to the IC stub
  // which will call the target, the stub will take care of the increment.
  const bool call_target_directly =
      miss_handler_ == MissHandler::kInlineCacheMiss;
  const intptr_t invocation_count = call_target_directly ? 1 : 0;

  if (caller_arguments_.length() == 1) {
    auto exactness = StaticTypeExactnessState::NotTracking();
#if !defined(DART_PRECOMPILED_RUNTIME)
    if (ic_data.is_tracking_exactness()) {
      exactness = receiver().IsNull()
                      ? StaticTypeExactnessState::NotExact()
                      : StaticTypeExactnessState::Compute(
                            Type::Cast(AbstractType::Handle(
                                ic_data.receivers_static_type())),
                            receiver());
    }
#endif  // !defined(DART_PRECOMPILED_RUNTIME)
    ic_data.EnsureHasReceiverCheck(receiver().GetClassId(), target_function,
                                   invocation_count, exactness);
  } else {
    GrowableArray<intptr_t> class_ids(caller_arguments_.length());
    ASSERT(ic_data.NumArgsTested() == caller_arguments_.length());
    for (intptr_t i = 0; i < caller_arguments_.length(); i++) {
      class_ids.Add(caller_arguments_[i]->GetClassId());
    }
    ic_data.EnsureHasCheck(class_ids, target_function, invocation_count);
  }
}

void PatchableCallHandler::ReturnAOT(const Code& stub, const Object& data) {
  ASSERT(miss_handler_ == MissHandler::kSwitchableCallMiss);
  arguments_.SetArgAt(0, stub);  // Second return value.
  arguments_.SetReturn(data);
}

void PatchableCallHandler::ReturnJIT(const Code& stub,
                                     const Object& data,
                                     const Function& target) {
  // In JIT we can have two different miss handlers to which we return slightly
  // differently.
  switch (miss_handler_) {
    case MissHandler::kSwitchableCallMiss: {
      arguments_.SetArgAt(0, stub);  // Second return value.
      arguments_.SetReturn(data);
      break;
    }
    case MissHandler::kFixCallersTargetMonomorphic: {
      arguments_.SetArgAt(1, data);  // Second return value.
      arguments_.SetReturn(stub);
      break;
    }
    case MissHandler::kInlineCacheMiss: {
      arguments_.SetReturn(target);
      break;
    }
  }
}

void PatchableCallHandler::ReturnJITorAOT(const Code& stub,
                                          const Object& data,
                                          const Function& target) {
#if defined(DART_PRECOMPILED_MODE)
  ReturnAOT(stub, data);
#else
  ReturnJIT(stub, data, target);
#endif
}

ICDataPtr PatchableCallHandler::NewICData() {
  return ICData::New(caller_function_, name_, args_descriptor_, DeoptId::kNone,
                     /*num_args_tested=*/1, ICData::kInstance);
}

ICDataPtr PatchableCallHandler::NewICDataWithTarget(intptr_t cid,
                                                    const Function& target) {
  GrowableArray<intptr_t> cids(1);
  cids.Add(cid);
  return ICData::NewWithCheck(caller_function_, name_, args_descriptor_,
                              DeoptId::kNone, /*num_args_tested=*/1,
                              ICData::kInstance, &cids, target);
}

FunctionPtr PatchableCallHandler::ResolveTargetFunction(const Object& data) {
  switch (data.GetClassId()) {
    case kUnlinkedCallCid: {
      const auto& unlinked_call = UnlinkedCall::Cast(data);

#if defined(DART_PRECOMPILED_RUNTIME)
      // When transitioning out of UnlinkedCall to other states (e.g.
      // Monomorphic, MonomorphicSmiable, SingleTarget) we lose
      // name/arg-descriptor in AOT mode and cannot recover it.
      //
      // Even if we could recover an old target function (which was missed) -
      // which we cannot in AOT bare mode - we can still lose the name due to a
      // dyn:* call site potentially targeting non-dyn:* targets.
      //
      // => We will therefore retain the unlinked call here.
      //
      // In JIT mode we always use ICData from the call site, which has the
      // correct name/args-descriptor.
      SaveUnlinkedCall(zone_, isolate_group_, caller_frame_->pc(),
                       unlinked_call);
#endif  // defined(DART_PRECOMPILED_RUNTIME)

      name_ = unlinked_call.target_name();
      args_descriptor_ = unlinked_call.arguments_descriptor();
      break;
    }
    case kMonomorphicSmiableCallCid:
      FALL_THROUGH;
#if defined(DART_PRECOMPILED_RUNTIME)
    case kSmiCid:
      FALL_THROUGH;
    case kSingleTargetCacheCid: {
      const auto& unlinked_call = UnlinkedCall::Handle(
          zone_, LoadUnlinkedCall(zone_, isolate_group_, caller_frame_->pc()));
      name_ = unlinked_call.target_name();
      args_descriptor_ = unlinked_call.arguments_descriptor();
      break;
    }
#else
    case kArrayCid: {
      // Monomorphic calls use the ICData::entries() as their data.
      const auto& ic_data_entries = Array::Cast(data);
      // Any non-empty ICData::entries() has a backref to it's ICData.
      const auto& ic_data =
          ICData::Handle(zone_, ICData::ICDataOfEntriesArray(ic_data_entries));
      args_descriptor_ = ic_data.arguments_descriptor();
      name_ = ic_data.target_name();
      break;
    }
#endif  // defined(DART_PRECOMPILED_RUNTIME)
    case kICDataCid:
      FALL_THROUGH;
    case kMegamorphicCacheCid: {
      const CallSiteData& call_site_data = CallSiteData::Cast(data);
      name_ = call_site_data.target_name();
      args_descriptor_ = call_site_data.arguments_descriptor();
      break;
    }
    default:
      UNREACHABLE();
  }
  const Class& cls = Class::Handle(zone_, receiver().clazz());
  return Resolve(thread_, zone_, caller_arguments_, cls, name_,
                 args_descriptor_, /*allow_add=*/!FLAG_precompiled_mode);
}

void PatchableCallHandler::ResolveSwitchAndReturn(const Object& old_data) {
  // Find out actual target (which can be time consuming) without holding any
  // locks.
  const auto& target_function =
      Function::Handle(zone_, ResolveTargetFunction(old_data));

  auto& data = Object::Handle(zone_);

  // We ensure any transition in a patchable calls are done in an atomic
  // manner, we ensure we always transition forward (e.g. Monomorphic ->
  // Polymorphic).
  //
  // Mutators are only stopped if we actually need to patch a patchable call.
  // We may not do that if we e.g. just add one more check to an ICData.
  SafepointMutexLocker ml(isolate_group_->patchable_call_mutex());

#if defined(DART_PRECOMPILED_RUNTIME)
  data =
      CodePatcher::GetSwitchableCallDataAt(caller_frame_->pc(), caller_code_);
  uword target_entry = 0;
  DEBUG_ONLY(target_entry = CodePatcher::GetSwitchableCallTargetEntryAt(
                 caller_frame_->pc(), caller_code_));
  HandleMissAOT(data, target_entry, target_function);
#else
  auto& code = Code::Handle(zone_);
  if (should_consider_patching()) {
    code ^= CodePatcher::GetInstanceCallAt(caller_frame_->pc(), caller_code_,
                                           &data);
  } else {
    ASSERT(old_data.IsICData() || old_data.IsMegamorphicCache());
    data = old_data.ptr();
  }
  HandleMissJIT(data, code, target_function);
#endif
}

#if defined(DART_PRECOMPILED_RUNTIME)

void PatchableCallHandler::HandleMissAOT(const Object& old_data,
                                         uword old_entry,
                                         const Function& target_function) {
  switch (old_data.GetClassId()) {
    case kUnlinkedCallCid:
      ASSERT(old_entry ==
             StubCode::SwitchableCallMiss().MonomorphicEntryPoint());
      DoUnlinkedCallAOT(UnlinkedCall::Cast(old_data), target_function);
      break;
    case kMonomorphicSmiableCallCid:
      ASSERT(old_entry ==
             StubCode::MonomorphicSmiableCheck().MonomorphicEntryPoint());
      FALL_THROUGH;
    case kSmiCid:
      DoMonomorphicMissAOT(old_data, target_function);
      break;
    case kSingleTargetCacheCid:
      ASSERT(old_entry == StubCode::SingleTargetCall().MonomorphicEntryPoint());
      DoSingleTargetMissAOT(SingleTargetCache::Cast(old_data), target_function);
      break;
    case kICDataCid:
      ASSERT(old_entry ==
             StubCode::ICCallThroughCode().MonomorphicEntryPoint());
      DoICDataMissAOT(ICData::Cast(old_data), target_function);
      break;
    case kMegamorphicCacheCid:
      ASSERT(old_entry == StubCode::MegamorphicCall().MonomorphicEntryPoint());
      DoMegamorphicMiss(MegamorphicCache::Cast(old_data), target_function);
      break;
    default:
      UNREACHABLE();
  }
}

#else

void PatchableCallHandler::HandleMissJIT(const Object& old_data,
                                         const Code& old_code,
                                         const Function& target_function) {
  switch (old_data.GetClassId()) {
    case kArrayCid:
      // ICData three-element array: Smi(receiver CID), Smi(count),
      // Function(target). It is the Array from ICData::entries_.
      DoMonomorphicMissJIT(old_data, target_function);
      break;
    case kICDataCid:
      DoICDataMissJIT(ICData::Cast(old_data), old_code, target_function);
      break;
    case kMegamorphicCacheCid:
      ASSERT(old_code.ptr() == StubCode::MegamorphicCall().ptr() ||
             (old_code.IsNull() && !should_consider_patching()));
      DoMegamorphicMiss(MegamorphicCache::Cast(old_data), target_function);
      break;
    default:
      UNREACHABLE();
  }
}
#endif  // defined(DART_PRECOMPILED_RUNTIME)

static void InlineCacheMissHandler(Thread* thread,
                                   Zone* zone,
                                   const GrowableArray<const Instance*>& args,
                                   const ICData& ic_data,
                                   NativeArguments native_arguments) {
#if !defined(DART_PRECOMPILED_RUNTIME)
  DartFrameIterator iterator(thread,
                             StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* caller_frame = iterator.NextFrame();
  const auto& caller_code = Code::Handle(zone, caller_frame->LookupDartCode());
  const auto& caller_function =
      Function::Handle(zone, caller_frame->LookupDartFunction());

  PatchableCallHandler handler(thread, args, MissHandler::kInlineCacheMiss,
                               native_arguments, caller_frame, caller_code,
                               caller_function);

  handler.ResolveSwitchAndReturn(ic_data);
#else
  UNREACHABLE();
#endif  // !defined(DART_PRECOMPILED_RUNTIME)
}

// Handles inline cache misses by updating the IC data array of the call site.
//   Arg0: Receiver object.
//   Arg1: IC data object.
//   Returns: target function with compiled code or null.
// Modifies the instance call to hold the updated IC data array.
DEFINE_RUNTIME_ENTRY(InlineCacheMissHandlerOneArg, 2) {
  const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const ICData& ic_data = ICData::CheckedHandle(zone, arguments.ArgAt(1));
  RELEASE_ASSERT(!FLAG_precompiled_mode);
  GrowableArray<const Instance*> args(1);
  args.Add(&receiver);
  InlineCacheMissHandler(thread, zone, args, ic_data, arguments);
}

// Handles inline cache misses by updating the IC data array of the call site.
//   Arg0: Receiver object.
//   Arg1: Argument after receiver.
//   Arg2: IC data object.
//   Returns: target function with compiled code or null.
// Modifies the instance call to hold the updated IC data array.
DEFINE_RUNTIME_ENTRY(InlineCacheMissHandlerTwoArgs, 3) {
  const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const Instance& other = Instance::CheckedHandle(zone, arguments.ArgAt(1));
  const ICData& ic_data = ICData::CheckedHandle(zone, arguments.ArgAt(2));
  RELEASE_ASSERT(!FLAG_precompiled_mode);
  GrowableArray<const Instance*> args(2);
  args.Add(&receiver);
  args.Add(&other);
  InlineCacheMissHandler(thread, zone, args, ic_data, arguments);
}

// Handle the first use of an instance call
//   Arg1: Receiver.
//   Arg0: Stub out.
//   Returns: the ICData used to continue with the call.
DEFINE_RUNTIME_ENTRY(SwitchableCallMiss, 2) {
  const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(1));

  StackFrameIterator iterator(ValidationPolicy::kDontValidateFrames, thread,
                              StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* exit_frame = iterator.NextFrame();
  ASSERT(exit_frame->IsExitFrame());
  StackFrame* miss_handler_frame = iterator.NextFrame();
  // This runtime entry can be called either from miss stub or from
  // switchable_call_miss "dart" stub/function set up in
  // [MegamorphicCacheTable::InitMissHandler].
  ASSERT(miss_handler_frame->IsStubFrame() ||
         miss_handler_frame->IsDartFrame());
  StackFrame* caller_frame = iterator.NextFrame();
  ASSERT(caller_frame->IsDartFrame());
  const Code& caller_code = Code::Handle(zone, caller_frame->LookupDartCode());
  const Function& caller_function =
      Function::Handle(zone, caller_frame->LookupDartFunction());

  auto& old_data = Object::Handle(zone);
#if defined(DART_PRECOMPILED_RUNTIME)
  old_data =
      CodePatcher::GetSwitchableCallDataAt(caller_frame->pc(), caller_code);
#else
  CodePatcher::GetInstanceCallAt(caller_frame->pc(), caller_code, &old_data);
#endif

  GrowableArray<const Instance*> caller_arguments(1);
  caller_arguments.Add(&receiver);
  PatchableCallHandler handler(thread, caller_arguments,
                               MissHandler::kSwitchableCallMiss, arguments,
                               caller_frame, caller_code, caller_function);
  handler.ResolveSwitchAndReturn(old_data);
}

// Handles interpreted interface call cache miss.
//   Arg0: receiver
//   Arg1: target name
//   Arg2: arguments descriptor
//   Returns: target function (can only be null in AOT runtime)
// Modifies the instance call table in current interpreter.
DEFINE_RUNTIME_ENTRY(InterpretedInstanceCallMissHandler, 3) {
#if defined(DART_DYNAMIC_MODULES)
  const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const String& target_name = String::CheckedHandle(zone, arguments.ArgAt(1));
  const Array& arg_desc = Array::CheckedHandle(zone, arguments.ArgAt(2));

  ArgumentsDescriptor arguments_descriptor(arg_desc);
  const Class& receiver_class = Class::Handle(zone, receiver.clazz());
  Function& target_function = Function::Handle(zone);
  if (receiver_class.EnsureIsFinalized(thread) == Error::null()) {
    const Class& cls = Class::Handle(zone, receiver.clazz());
    // Unlike compiled AOT, lazily create dynamic invocation forwarders if
    // not created during the precompiler (e.g., the only dynamic calls are
    // in interpreted code). This can be done for both compiled and
    // interpreted code, as the resulting forwarder will be interpreted.
    const bool allow_add = true;
    target_function = Resolver::ResolveDynamicForReceiverClass(
        cls, target_name, arguments_descriptor, allow_add);
  }

  // TODO(regis): In order to substitute 'simple_instance_of_function', the 2nd
  // arg to the call, the type, is needed.

  if (target_function.IsNull()) {
    target_function =
        InlineCacheMissHelper(receiver_class, arg_desc, target_name,
                              /*allow_add=*/!FLAG_precompiled_mode);
  }
#if !defined(DART_PRECOMPILED_RUNTIME)
  ASSERT(!target_function.IsNull());
#endif
  arguments.SetReturn(target_function);
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES)
}

// Used to find the correct receiver and function to invoke or to fall back to
// invoking noSuchMethod when lazy dispatchers are disabled. Returns the
// result of the invocation or an Error.
static ObjectPtr InvokeCallThroughGetterOrNoSuchMethod(
    Thread* thread,
    Zone* zone,
    const Instance& receiver,
    const String& target_name,
    const Array& orig_arguments,
    const Array& orig_arguments_desc) {
  const bool is_dynamic_call =
      Function::IsDynamicInvocationForwarderName(target_name);
  const bool is_dyn_implicit_call =
      target_name.ptr() == Symbols::DynamicImplicitCall().ptr();
  String& demangled_target_name = String::Handle(zone, target_name.ptr());
  if (is_dynamic_call) {
    demangled_target_name =
        Function::DemangleDynamicInvocationForwarderName(target_name);
  }
  Class& cls = Class::Handle(zone, receiver.clazz());
  Function& function = Function::Handle(zone);

  // Dart distinguishes getters and regular methods and allows their calls
  // to mix with conversions, and its selectors are independent of arity. So do
  // a zigzagged lookup to see if this call failed because of an arity mismatch,
  // need for conversion, or there really is no such method.

  const bool is_getter = Field::IsGetterName(demangled_target_name);
  if (is_getter) {
    // Tear-off of a method
    // o.foo (o.get:foo) failed, closurize o.foo() if it exists.
    const auto& function_name =
        String::Handle(zone, Field::NameFromGetter(demangled_target_name));
    while (!cls.IsNull()) {
      // We don't generate dyn:* forwarders for method extractors so there is no
      // need to try to find a dyn:get:foo first.
      if (function.IsNull()) {
        if (cls.EnsureIsFinalized(thread) == Error::null()) {
          function = Resolver::ResolveDynamicFunction(zone, cls, function_name);
        }
      }
      if (!function.IsNull()) {
        const Function& closure_function =
            Function::Handle(zone, function.ImplicitClosureFunction());
        const Object& result = Object::Handle(
            zone, closure_function.ImplicitInstanceClosure(receiver));
        return result.ptr();
      }
      cls = cls.SuperClass();
    }

    if (receiver.IsRecord()) {
      const Record& record = Record::Cast(receiver);
      const intptr_t field_index =
          record.GetFieldIndexByName(thread, function_name);
      if (field_index >= 0) {
        return record.FieldAt(field_index);
      }
    }

    // Fall through for noSuchMethod
  } else {
    // Call through field.
    // o.foo(...) failed, invoke noSuchMethod is foo exists but has the wrong
    // number of arguments, or try (o.foo).call(...)

    if ((demangled_target_name.ptr() == Symbols::call().ptr()) &&
        receiver.IsClosure()) {
      // Special case: closures are implemented with a call getter instead of a
      // call method and with lazy dispatchers the field-invocation-dispatcher
      // would perform the closure call.
      return DartEntry::InvokeClosure(thread, orig_arguments,
                                      orig_arguments_desc);
    }

    // Dynamic call sites have to use the dynamic getter as well (if it was
    // created), unless its a dynamic implicit call to 'call'.
    const auto& getter_name =
        String::Handle(zone, Field::GetterName(demangled_target_name));
    const auto& dyn_getter_name = String::Handle(
        zone, is_dynamic_call
                  ? Function::CreateDynamicInvocationForwarderName(getter_name)
                  : getter_name.ptr());
    ArgumentsDescriptor args_desc(orig_arguments_desc);
    while (!cls.IsNull()) {
      // If there is a function with the target name but mismatched arguments
      // we need to call `receiver.noSuchMethod()`. Similarly, if there is a
      // function that we aren't allowed to invoke because the target function
      // was not dynamically-callable from a dynamic module.
      if (cls.EnsureIsFinalized(thread) == Error::null()) {
        function = Resolver::ResolveDynamicFunction(zone, cls, target_name);
      }
      if (!function.IsNull()) {
        ASSERT(!function.is_dynamically_callable() ||
               !function.AreValidArguments(args_desc, nullptr));
        break;  // mismatch, invoke noSuchMethod
      }
      if (is_dynamic_call) {
        function =
            Resolver::ResolveDynamicFunction(zone, cls, demangled_target_name);
        if (!function.IsNull()) {
          ASSERT(!function.is_dynamically_callable() ||
                 !function.AreValidArguments(args_desc, nullptr));
          break;  // mismatch, invoke noSuchMethod
        }
      }

      if (!is_dyn_implicit_call) {
        // If there is a getter we need to call-through-getter.
        if (is_dynamic_call) {
          function =
              Resolver::ResolveDynamicFunction(zone, cls, dyn_getter_name);
        }
        if (function.IsNull()) {
          function = Resolver::ResolveDynamicFunction(zone, cls, getter_name);
        }
        if (!function.IsNull()) {
          const Array& getter_arguments = Array::Handle(Array::New(1));
          getter_arguments.SetAt(0, receiver);
          const Object& getter_result = Object::Handle(
              zone, DartEntry::InvokeFunction(function, getter_arguments));
          if (getter_result.IsError()) {
            return getter_result.ptr();
          }
          ASSERT(getter_result.IsNull() || getter_result.IsInstance());

          orig_arguments.SetAt(args_desc.FirstArgIndex(), getter_result);
          return DartEntry::InvokeClosure(thread, orig_arguments,
                                          orig_arguments_desc);
        }
      }
      cls = cls.SuperClass();
    }

    if (receiver.IsRecord() && !is_dyn_implicit_call) {
      const Record& record = Record::Cast(receiver);
      const intptr_t field_index =
          record.GetFieldIndexByName(thread, demangled_target_name);
      if (field_index >= 0) {
        const Object& getter_result =
            Object::Handle(zone, record.FieldAt(field_index));
        ASSERT(getter_result.IsNull() || getter_result.IsInstance());
        orig_arguments.SetAt(args_desc.FirstArgIndex(), getter_result);
        return DartEntry::InvokeClosure(thread, orig_arguments,
                                        orig_arguments_desc);
      }
    }
  }

  const Object& result = Object::Handle(
      zone,
      DartEntry::InvokeNoSuchMethod(thread, receiver, demangled_target_name,
                                    orig_arguments, orig_arguments_desc));
  return result.ptr();
}

// Invoke appropriate noSuchMethod or closure from getter.
// Arg0: receiver
// Arg1: ICData or MegamorphicCache
// Arg2: arguments descriptor array
// Arg3: arguments array
DEFINE_RUNTIME_ENTRY(NoSuchMethodFromCallStub, 4) {
  const Object& ic_data_or_cache = Object::Handle(zone, arguments.ArgAt(1));
  String& target_name = String::Handle(zone);
  if (ic_data_or_cache.IsICData()) {
    target_name = ICData::Cast(ic_data_or_cache).target_name();
  } else {
    ASSERT(ic_data_or_cache.IsMegamorphicCache());
    target_name = MegamorphicCache::Cast(ic_data_or_cache).target_name();
  }
#if defined(DART_PRECOMPILED_RUNTIME)
  const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const Array& orig_arguments_desc =
      Array::CheckedHandle(zone, arguments.ArgAt(2));
  const Array& orig_arguments = Array::CheckedHandle(zone, arguments.ArgAt(3));
  const auto& result =
      Object::Handle(zone, InvokeCallThroughGetterOrNoSuchMethod(
                               thread, zone, receiver, target_name,
                               orig_arguments, orig_arguments_desc));
  ThrowIfError(result);
  arguments.SetReturn(result);
#else
  FATAL("Dispatcher for %s should have been lazily created",
        target_name.ToCString());
#endif
}

#if !defined(DART_PRECOMPILED_RUNTIME)
static ArrayPtr CollectArgumentsToArray(Thread* thread,
                                        Zone* zone,
                                        const Array& args_descriptor) {
  DartFrameIterator iterator(thread,
                             StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* caller_frame = iterator.NextFrame();
  ObjectPtr* argv = reinterpret_cast<ObjectPtr*>(caller_frame->sp());

  ArgumentsDescriptor args_desc(args_descriptor);
  const Array& args =
      Array::Handle(zone, Array::New(args_desc.CountWithTypeArgs()));
  auto& arg = Object::Handle(zone);
  for (intptr_t i = args.Length() - 1; i >= 0; --i) {
    arg = *argv++;
    args.SetAt(i, arg);
  }
  return args.ptr();
}

static void PatchTypeArgumentsForDynamicCallModAOT(
    Thread* thread,
    Zone* zone,
    const Array& args_descriptor,
    const TypeArguments& default_type_args) {
  DartFrameIterator iterator(thread,
                             StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* caller_frame = iterator.NextFrame();
  ObjectPtr* argv = reinterpret_cast<ObjectPtr*>(caller_frame->sp());
  ArgumentsDescriptor args_desc(args_descriptor);
  argv = argv + args_desc.Count();
  // Dynamic calls in modular AOT always provide type arguments or
  // reserve a slot for type arguments.
  RELEASE_ASSERT(*argv == Object::null());
  *argv = default_type_args.ptr();
}
#endif  // !defined(DART_PRECOMPILED_RUNTIME)

// Handles inline cache misses in the modular AOT.
// Resolves the target function and update the IC data array if found.
// Also handles calls through getters and noSuchMethod.
//   Arg0: Placeholder for the result of the call.
//   Arg1: Receiver object.
//   Arg2: IC data object.
//   Returns: target function or null if call was handled.
DEFINE_RUNTIME_ENTRY(InlineCacheMissHandlerModAOT, 3) {
#if !defined(DART_PRECOMPILED_RUNTIME)
  const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(1));
  const ICData& ic_data = ICData::CheckedHandle(zone, arguments.ArgAt(2));
  RELEASE_ASSERT(!FLAG_precompiled_mode);
  GrowableArray<const Instance*> args(1);
  args.Add(&receiver);

  const auto& name = String::Handle(zone, ic_data.target_name());
  const auto& args_descriptor =
      Array::Handle(zone, ic_data.arguments_descriptor());
  const auto& cls = Class::Handle(zone, receiver.clazz());
  const auto& target_function =
      Function::Handle(zone, Resolve(thread, zone, args, cls, name,
                                     args_descriptor, /*allow_add=*/true));
  arguments.SetReturn(target_function);

  if (!target_function.IsNull()) {
    ic_data.EnsureHasReceiverCheck(cls.id(), target_function);
    return;
  }

  ArgumentsDescriptor args_desc(args_descriptor);
  const Array& orig_arguments = Array::Handle(
      zone, CollectArgumentsToArray(thread, zone, args_descriptor));
  ASSERT(orig_arguments.At(args_desc.FirstArgIndex()) == receiver.ptr());

  const auto& result = Object::Handle(
      zone, InvokeCallThroughGetterOrNoSuchMethod(
                thread, zone, receiver, name, orig_arguments, args_descriptor));
  ThrowIfError(result);
  arguments.SetArgAt(0, result);
#else
  UNREACHABLE();
#endif  // !defined(DART_PRECOMPILED_RUNTIME)
}

// Check that argument types are valid for the given dynamic invocation
// forwarder function. Pass default type arguments if needed.
// Arg0: dynamic invocation forwarder function
// Arg1: arguments descriptor
// Return value: target function.
DEFINE_RUNTIME_ENTRY(DynamicInvocationForwarderModAOT, 2) {
#if !defined(DART_PRECOMPILED_RUNTIME)
  const auto& function = Function::CheckedHandle(zone, arguments.ArgAt(0));
  const auto& args_descriptor = Array::CheckedHandle(zone, arguments.ArgAt(1));
  const auto& target = Function::Handle(zone, function.ForwardingTarget());

  ArgumentsDescriptor args_desc(args_descriptor);
  const Array& args = Array::Handle(
      zone, CollectArgumentsToArray(thread, zone, args_descriptor));

  // Ensured during resolution.
  ASSERT(target.AreValidArguments(args_desc, nullptr));

  const auto& result =
      Object::Handle(zone, target.DoArgumentTypesMatch(args, args_desc));
  if (result.IsError()) {
    Exceptions::PropagateError(Error::Cast(result));
  }

  if (target.IsGeneric() && (args_desc.TypeArgsLen() == 0)) {
    // Adjust arguments descriptor to pass default type arguments.
    auto& default_type_args =
        TypeArguments::Handle(zone, target.DefaultTypeArguments(zone));
    if (!default_type_args.IsNull() && !default_type_args.IsInstantiated()) {
      ASSERT(!target.is_static());
      const auto& receiver =
          Object::Handle(zone, args.At(args_desc.FirstArgIndex()));
      const auto& instantiator_type_arguments = TypeArguments::Handle(
          zone, Instance::Cast(receiver).GetTypeArguments());
      default_type_args = default_type_args.InstantiateFrom(
          instantiator_type_arguments, Object::null_type_arguments(), kAllFree,
          Heap::kOld);
    }
    const auto& arg_names = Array::Handle(zone, args_desc.GetArgumentNames());
    const auto& new_args_desc = Array::Handle(
        zone, ArgumentsDescriptor::NewBoxed(target.NumTypeArguments(),
                                            args_desc.Count(), arg_names));
    arguments.SetArgAt(1, new_args_desc);
    PatchTypeArgumentsForDynamicCallModAOT(thread, zone, new_args_desc,
                                           default_type_args);
  }
  arguments.SetReturn(target);
#else
  UNREACHABLE();
#endif  // !defined(DART_PRECOMPILED_RUNTIME)
}

// Invoke appropriate noSuchMethod function.
// Arg0: receiver
// Arg1: function
// Arg1: arguments descriptor array.
// Arg3: arguments array.
DEFINE_RUNTIME_ENTRY(NoSuchMethodFromPrologue, 4) {
  const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const Function& function = Function::CheckedHandle(zone, arguments.ArgAt(1));
  const Array& orig_arguments_desc =
      Array::CheckedHandle(zone, arguments.ArgAt(2));
  const Array& orig_arguments = Array::CheckedHandle(zone, arguments.ArgAt(3));

  String& orig_function_name = String::Handle(zone);
  if ((function.kind() == UntaggedFunction::kClosureFunction) ||
      (function.kind() == UntaggedFunction::kImplicitClosureFunction)) {
    // For closure the function name is always 'call'. Replace it with the
    // name of the closurized function so that exception contains more
    // relevant information.
    orig_function_name = function.QualifiedUserVisibleName();
  } else {
    orig_function_name = function.name();
  }

  const Object& result = Object::Handle(
      zone, DartEntry::InvokeNoSuchMethod(thread, receiver, orig_function_name,
                                          orig_arguments, orig_arguments_desc));
  ThrowIfError(result);
  arguments.SetReturn(result);
}

// Throw NoSuchMethodError with given arguments.
// Arg0: arguments of NoSuchMethodError._throwNew.
DEFINE_RUNTIME_ENTRY(NoSuchMethodError, 1) {
  const Array& args = Array::CheckedHandle(zone, arguments.ArgAt(0));
  const Library& libcore = Library::Handle(Library::CoreLibrary());
  const Class& cls =
      Class::Handle(libcore.LookupClass(Symbols::NoSuchMethodError()));
  ASSERT(!cls.IsNull());
  const auto& error = cls.EnsureIsFinalized(Thread::Current());
  ASSERT(error == Error::null());
  const Function& throwNew =
      Function::Handle(cls.LookupFunctionAllowPrivate(Symbols::ThrowNew()));
  ASSERT(args.Length() == throwNew.NumParameters());
  const Object& result =
      Object::Handle(zone, DartEntry::InvokeFunction(throwNew, args));
  ThrowIfError(result);
  arguments.SetReturn(result);
}

// Invoke appropriate noSuchMethod function (or in the case of no lazy
// dispatchers, walk the receiver to find the correct method to call).
// Arg0: receiver
// Arg1: function name.
// Arg2: arguments descriptor array.
// Arg3: arguments array.
DEFINE_RUNTIME_ENTRY(InvokeNoSuchMethod, 4) {
#if defined(DART_DYNAMIC_MODULES)
  const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const String& original_function_name =
      String::CheckedHandle(zone, arguments.ArgAt(1));
  const Array& orig_arguments_desc =
      Array::CheckedHandle(zone, arguments.ArgAt(2));
  const Array& orig_arguments = Array::CheckedHandle(zone, arguments.ArgAt(3));

  auto& result = Object::Handle(zone);
#if defined(DART_PRECOMPILED_RUNTIME)
  // Failing to find the method could be due to the lack of lazy invoke field
  // dispatchers, so attempt a deeper search before calling noSuchMethod.
  result = InvokeCallThroughGetterOrNoSuchMethod(
      thread, zone, receiver, original_function_name, orig_arguments,
      orig_arguments_desc);
#else
  result =
      DartEntry::InvokeNoSuchMethod(thread, receiver, original_function_name,
                                    orig_arguments, orig_arguments_desc);
#endif
  ThrowIfError(result);
  arguments.SetReturn(result);
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES)
}

#if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
// The following code is used to stress test
//  - deoptimization
//  - debugger stack tracing
//  - garbage collection
//  - hot reload
static void HandleStackOverflowTestCases(Thread* thread) {
  auto isolate_group = thread->isolate_group();

  if (FLAG_shared_slow_path_triggers_gc) {
    isolate_group->heap()->CollectAllGarbage(GCReason::kDebugging);
  }

  bool do_deopt = false;
  bool do_stacktrace = false;
  bool do_reload = false;
  bool do_gc = false;
  const intptr_t isolate_reload_every =
      isolate_group->reload_every_n_stack_overflow_checks();
  if ((FLAG_deoptimize_every > 0) || (FLAG_stacktrace_every > 0) ||
      (FLAG_gc_every > 0) || (isolate_reload_every > 0)) {
    if (!IsolateGroup::IsSystemIsolateGroup(isolate_group)) {
      // TODO(turnidge): To make --deoptimize_every and
      // --stacktrace-every faster we could move this increment/test to
      // the generated code.
      uint32_t count = thread->IncrementAndGetStackOverflowCount();
      if (FLAG_deoptimize_every > 0 && (count % FLAG_deoptimize_every) == 0) {
        do_deopt = true;
      }
      if (FLAG_stacktrace_every > 0 && (count % FLAG_stacktrace_every) == 0) {
        do_stacktrace = true;
      }
      if (FLAG_gc_every > 0 && (count % FLAG_gc_every) == 0) {
        do_gc = true;
      }
      if ((isolate_reload_every > 0) && (count % isolate_reload_every) == 0) {
        do_reload =
            isolate_group->CanReload() && !isolate_group->has_seen_oom();
      }
    }
  }
  if ((FLAG_deoptimize_filter != nullptr) ||
      (FLAG_stacktrace_filter != nullptr) || (FLAG_reload_every != 0)) {
    DartFrameIterator iterator(thread,
                               StackFrameIterator::kNoCrossThreadIteration);
    StackFrame* frame = iterator.NextFrame();
    ASSERT(frame != nullptr);
    Code& code = Code::Handle();
    Function& function = Function::Handle();
    if (frame->is_interpreted()) {
      function = frame->LookupDartFunction();
    } else {
      code = frame->LookupDartCode();
      ASSERT(!code.IsNull());
      function = code.function();
    }
    ASSERT(!function.IsNull());
    const char* function_name = nullptr;
    if ((FLAG_deoptimize_filter != nullptr) ||
        (FLAG_stacktrace_filter != nullptr)) {
      function_name = function.ToFullyQualifiedCString();
      ASSERT(function_name != nullptr);
    }
    if (!code.IsNull()) {
      if (!code.is_optimized() && FLAG_reload_every_optimized) {
        // Don't do the reload if we aren't inside optimized code.
        do_reload = false;
      }
      if (code.is_optimized() && FLAG_deoptimize_filter != nullptr &&
          strstr(function_name, FLAG_deoptimize_filter) != nullptr &&
          !function.ForceOptimize()) {
        OS::PrintErr("*** Forcing deoptimization (%s)\n",
                     function.ToFullyQualifiedCString());
        do_deopt = true;
      }
    }
    if (FLAG_stacktrace_filter != nullptr &&
        strstr(function_name, FLAG_stacktrace_filter) != nullptr) {
      OS::PrintErr("*** Computing stacktrace (%s)\n",
                   function.ToFullyQualifiedCString());
      do_stacktrace = true;
    }
  }
  if (do_deopt) {
    // TODO(turnidge): Consider using DeoptimizeAt instead.
    DeoptimizeFunctionsOnStack();
  }
  if (do_reload) {
    // Maybe adjust the rate of future reloads.
    isolate_group->MaybeIncreaseReloadEveryNStackOverflowChecks();

    // Issue a reload.
    const char* script_uri = isolate_group->source()->script_uri;
    JSONStream js;
    const bool success =
        isolate_group->ReloadSources(&js, /*force_reload=*/true, script_uri);
    if (!success && !Dart::IsShuttingDown() && !isolate_group->has_seen_oom()) {
      FATAL("*** Isolate reload failed:\n%s\n", js.ToCString());
    }
  }
  if (do_stacktrace) {
    String& var_name = String::Handle();
    Instance& var_value = Instance::Handle();
    DebuggerStackTrace* stack = DebuggerStackTrace::Collect();
    intptr_t num_frames = stack->Length();
    for (intptr_t i = 0; i < num_frames; i++) {
      ActivationFrame* frame = stack->FrameAt(i);
      int num_vars = 0;
      // Variable locations and number are unknown when precompiling.
#if !defined(DART_PRECOMPILED_RUNTIME)
      const auto& function = frame->function();
      if (!function.ForceOptimize()) {
        if (!function.is_declared_in_bytecode()) {
          // Ensure that we have unoptimized code.
          function.EnsureHasCompiledUnoptimizedCode();
        }
        num_vars = frame->NumLocalVariables();
      }
#endif
      TokenPosition unused = TokenPosition::kNoSource;
      for (intptr_t v = 0; v < num_vars; v++) {
        frame->VariableAt(v, &var_name, &unused, &unused, &unused, &var_value);
      }
    }
    if (FLAG_stress_async_stacks) {
      DebuggerStackTrace::CollectAsyncAwaiters();
    }
  }
  if (do_gc) {
    isolate_group->heap()->CollectAllGarbage(GCReason::kDebugging);
  }
}
#endif  // !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)

#if !defined(DART_PRECOMPILED_RUNTIME)
static void HandleOSRRequest(Thread* thread) {
  auto isolate_group = thread->isolate_group();
  ASSERT(isolate_group->use_osr());
  DartFrameIterator iterator(thread,
                             StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* frame = iterator.NextFrame();
  ASSERT(frame != nullptr);
  const Code& code = Code::ZoneHandle(frame->LookupDartCode());
  ASSERT(!code.IsNull());
  ASSERT(!code.is_optimized());
  const Function& function = Function::Handle(code.function());
  ASSERT(!function.IsNull());

  // If the code of the frame does not match the function's unoptimized code,
  // we bail out since the code was reset by an isolate reload.
  if (code.ptr() != function.unoptimized_code()) {
    return;
  }

  // Since the code is referenced from the frame and the ZoneHandle,
  // it cannot have been removed from the function.
  ASSERT(function.HasCode());
  // Don't do OSR on intrinsified functions: The intrinsic code expects to be
  // called like a regular function and can't be entered via OSR.
  if (!Compiler::CanOptimizeFunction(thread, function) ||
      function.is_intrinsic()) {
    return;
  }

  // The unoptimized code is on the stack and should never be detached from
  // the function at this point.
  ASSERT(function.unoptimized_code() != Object::null());
  intptr_t osr_id =
      Code::Handle(function.unoptimized_code()).GetDeoptIdForOsr(frame->pc());
  ASSERT(osr_id != Compiler::kNoOSRDeoptId);
  if (FLAG_trace_osr) {
    OS::PrintErr("Attempting OSR for %s at id=%" Pd ", count=%" Pd "\n",
                 function.ToFullyQualifiedCString(), osr_id,
                 function.usage_counter());
  }

  // Since the code is referenced from the frame and the ZoneHandle,
  // it cannot have been removed from the function.
  const Object& result = Object::Handle(
      Compiler::CompileOptimizedFunction(thread, function, osr_id));
  ThrowIfError(result);

  if (!result.IsNull()) {
    const Code& code = Code::Cast(result);
    uword optimized_entry = code.EntryPoint();
    frame->set_pc(optimized_entry);
    frame->set_pc_marker(code.ptr());
  }
}
#endif  // !defined(DART_PRECOMPILED_RUNTIME)

DEFINE_RUNTIME_ENTRY(InterruptOrStackOverflow, 0) {
  uword stack_pos = OSThread::GetCurrentStackPointer();
#if defined(DART_INCLUDE_SIMULATOR)
  if (FLAG_use_simulator) {
    stack_pos = Simulator::Current()->get_sp();
    // If simulator was never called it may return 0 as a value of SPREG.
    if (stack_pos == 0) {
      // Use any reasonable value which would not be treated
      // as stack overflow.
      stack_pos = thread->saved_stack_limit();
    }
  }
#endif
  // Always clear the stack overflow flags.  They are meant for this
  // particular stack overflow runtime call and are not meant to
  // persist.
  uword stack_overflow_flags = thread->GetAndClearStackOverflowFlags();

  bool interpreter_stack_overflow = false;
#if defined(DART_DYNAMIC_MODULES)
  Interpreter* interpreter = thread->interpreter();
  if (interpreter != nullptr) {
    interpreter_stack_overflow =
        interpreter->get_sp() >= interpreter->overflow_stack_limit();
  }
#endif  // defined(DART_DYNAMIC_MODULES)

  // If an interrupt happens at the same time as a stack overflow, we
  // process the stack overflow now and leave the interrupt for next
  // time.
  if (interpreter_stack_overflow || !thread->os_thread()->HasStackHeadroom() ||
      IsCalleeFrameOf(thread->saved_stack_limit(), stack_pos)) {
    if (FLAG_verbose_stack_overflow) {
      OS::PrintErr("Stack overflow\n");
      OS::PrintErr("  Native SP = %" Px ", stack limit = %" Px "\n", stack_pos,
                   thread->saved_stack_limit());
#if defined(DART_DYNAMIC_MODULES)
      if (thread->interpreter() != nullptr) {
        OS::PrintErr("  Interpreter SP = %" Px ", stack limit = %" Px "\n",
                     thread->interpreter()->get_sp(),
                     thread->interpreter()->overflow_stack_limit());
      }
#endif  // defined(DART_DYNAMIC_MODULES)

      OS::PrintErr("Call stack:\n");
      OS::PrintErr("size | frame\n");
      StackFrameIterator frames(ValidationPolicy::kDontValidateFrames, thread,
                                StackFrameIterator::kNoCrossThreadIteration);
      uword fp = stack_pos;
      StackFrame* frame = frames.NextFrame();
      while (frame != nullptr) {
        uword delta = (frame->fp() - fp);
        fp = frame->fp();
        OS::PrintErr("%4" Pd " %s\n", delta, frame->ToCString());
        frame = frames.NextFrame();
      }
    }

    // Use the preallocated stack overflow exception to avoid calling
    // into dart code.
    const Instance& exception = Instance::Handle(
        thread->isolate_group()->object_store()->stack_overflow());
    Exceptions::Throw(thread, exception);
    UNREACHABLE();
  }

#if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
  HandleStackOverflowTestCases(thread);
#endif  // !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)

  // Handle interrupts:
  //  - store buffer overflow
  //  - OOB message (vm-service or dart:isolate)
  //  - marking ready for finalization
  const Error& error = Error::Handle(thread->HandleInterrupts());
  ThrowIfError(error);

#if !defined(DART_PRECOMPILED_RUNTIME)
  if ((stack_overflow_flags & Thread::kOsrRequest) != 0) {
    HandleOSRRequest(thread);
  }
#else
  ASSERT((stack_overflow_flags & Thread::kOsrRequest) == 0);
#endif  // !defined(DART_PRECOMPILED_RUNTIME)
}

// Compile a function. Should call only if the function has not been compiled.
//   Arg0: function object.
DEFINE_RUNTIME_ENTRY(CompileFunction, 1) {
  ASSERT(thread->IsDartMutatorThread());

  {
    // Another isolate's mutator thread may have created [function] and
    // published it via an ICData, MegamorphicCache etc. Entering the lock below
    // is an acquire operation that pairs with the release operation when the
    // other isolate exited the lock, ensuring the initializing stores for
    // [function] are visible in the current thread.
    SafepointReadRwLocker ml(thread, thread->isolate_group()->program_lock());
  }

  // After the barrier, since this will read the object's header.
  const Function& function = Function::CheckedHandle(zone, arguments.ArgAt(0));

  // Will throw if compilation failed (e.g. with compile-time error).
  function.EnsureHasCode();
}

// This is called from function that needs to be optimized.
// The requesting function can be already optimized (reoptimization).
// Returns the Code object where to continue execution.
DEFINE_RUNTIME_ENTRY(OptimizeInvokedFunction, 1) {
#if !defined(DART_PRECOMPILED_RUNTIME)
  const Function& function = Function::CheckedHandle(zone, arguments.ArgAt(0));
  ASSERT(!function.IsNull());
  ASSERT(function.HasCode());

  if (Compiler::CanOptimizeFunction(thread, function)) {
    auto isolate_group = thread->isolate_group();
    if (FLAG_background_compilation) {
      if (isolate_group->background_compiler()->EnqueueCompilation(function)) {
        // Reduce the chance of triggering a compilation while the function is
        // being compiled in the background. INT32_MIN should ensure that it
        // takes long time to trigger a compilation.
        // Note that the background compilation queue rejects duplicate entries.
        function.SetUsageCounter(INT32_MIN);
        // Continue in the same code.
        arguments.SetReturn(function);
        return;
      }
    }

    // Reset usage counter for reoptimization before calling optimizer to
    // prevent recursive triggering of function optimization.
    function.SetUsageCounter(0);
    if (FLAG_trace_compiler || FLAG_trace_optimizing_compiler) {
      if (function.HasOptimizedCode()) {
        THR_Print("ReCompiling function: '%s' \n",
                  function.ToFullyQualifiedCString());
      }
    }
    Object& result = Object::Handle(
        zone, Compiler::CompileOptimizedFunction(thread, function));
    ThrowIfError(result);
  }
  arguments.SetReturn(function);
#else
  UNREACHABLE();
#endif  // !DART_PRECOMPILED_RUNTIME
}

// The caller must be a static call in a Dart frame, or an entry frame.
// Patch static call to point to valid code's entry point.
DEFINE_RUNTIME_ENTRY(FixCallersTarget, 0) {
#if !defined(DART_PRECOMPILED_RUNTIME)
  StackFrameIterator iterator(ValidationPolicy::kDontValidateFrames, thread,
                              StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* frame = iterator.NextFrame();
  ASSERT(frame != nullptr);
  while (frame->IsStubFrame() || frame->IsExitFrame()) {
    frame = iterator.NextFrame();
    ASSERT(frame != nullptr);
  }
  if (frame->IsEntryFrame()) {
    // Since function's current code is always unpatched, the entry frame always
    // calls to unpatched code.
    UNREACHABLE();
  }
  ASSERT(frame->IsDartFrame());
  const Code& caller_code = Code::Handle(zone, frame->LookupDartCode());
  RELEASE_ASSERT(caller_code.is_optimized());
  const Function& target_function = Function::Handle(
      zone, caller_code.GetStaticCallTargetFunctionAt(frame->pc()));

  const Code& current_target_code =
      Code::Handle(zone, target_function.EnsureHasCode());
  CodePatcher::PatchStaticCallAt(frame->pc(), caller_code, current_target_code);
  caller_code.SetStaticCallTargetCodeAt(frame->pc(), current_target_code);
  if (FLAG_trace_patching) {
    OS::PrintErr(
        "FixCallersTarget: caller %#" Px
        " "
        "target '%s' -> %#" Px " (%s)\n",
        frame->pc(), target_function.ToFullyQualifiedCString(),
        current_target_code.EntryPoint(),
        current_target_code.is_optimized() ? "optimized" : "unoptimized");
  }
  arguments.SetReturn(current_target_code);
#else
  UNREACHABLE();
#endif
}

// The caller must be a monomorphic call from unoptimized code.
// Patch call to point to new target.
DEFINE_RUNTIME_ENTRY(FixCallersTargetMonomorphic, 2) {
#if !defined(DART_PRECOMPILED_RUNTIME)
  const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const Array& switchable_call_data =
      Array::CheckedHandle(zone, arguments.ArgAt(1));

  DartFrameIterator iterator(thread,
                             StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* caller_frame = iterator.NextFrame();
  const auto& caller_code = Code::Handle(zone, caller_frame->LookupDartCode());
  const auto& caller_function =
      Function::Handle(zone, caller_frame->LookupDartFunction());

  GrowableArray<const Instance*> caller_arguments(1);
  caller_arguments.Add(&receiver);
  PatchableCallHandler handler(
      thread, caller_arguments, MissHandler::kFixCallersTargetMonomorphic,
      arguments, caller_frame, caller_code, caller_function);
  handler.ResolveSwitchAndReturn(switchable_call_data);
#else
  UNREACHABLE();
#endif
}

// The caller tried to allocate an instance via an invalidated allocation
// stub.
DEFINE_RUNTIME_ENTRY(FixAllocationStubTarget, 0) {
#if !defined(DART_PRECOMPILED_RUNTIME)
  StackFrameIterator iterator(ValidationPolicy::kDontValidateFrames, thread,
                              StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* frame = iterator.NextFrame();
  ASSERT(frame != nullptr);
  while (frame->IsStubFrame() || frame->IsExitFrame()) {
    frame = iterator.NextFrame();
    ASSERT(frame != nullptr);
  }
  if (frame->IsEntryFrame()) {
    // There must be a valid Dart frame.
    UNREACHABLE();
  }
  ASSERT(frame->IsDartFrame());
  const Code& caller_code = Code::Handle(zone, frame->LookupDartCode());
  ASSERT(!caller_code.IsNull());
  const Code& stub = Code::Handle(
      CodePatcher::GetStaticCallTargetAt(frame->pc(), caller_code));
  Class& alloc_class = Class::ZoneHandle(zone);
  alloc_class ^= stub.owner();
  Code& alloc_stub = Code::Handle(zone, alloc_class.allocation_stub());
  if (alloc_stub.IsNull()) {
    alloc_stub = StubCode::GetAllocationStubForClass(alloc_class);
    ASSERT(!alloc_stub.IsDisabled());
  }
  CodePatcher::PatchStaticCallAt(frame->pc(), caller_code, alloc_stub);
  caller_code.SetStubCallTargetCodeAt(frame->pc(), alloc_stub);
  if (FLAG_trace_patching) {
    OS::PrintErr("FixAllocationStubTarget: caller %#" Px
                 " alloc-class %s "
                 " -> %#" Px "\n",
                 frame->pc(), alloc_class.ToCString(), alloc_stub.EntryPoint());
  }
  arguments.SetReturn(alloc_stub);
#else
  UNREACHABLE();
#endif
}

const char* DeoptReasonToCString(ICData::DeoptReasonId deopt_reason) {
  switch (deopt_reason) {
#define DEOPT_REASON_TO_TEXT(name)                                             \
  case ICData::kDeopt##name:                                                   \
    return #name;
    DEOPT_REASONS(DEOPT_REASON_TO_TEXT)
#undef DEOPT_REASON_TO_TEXT
    default:
      UNREACHABLE();
      return "";
  }
}

static bool IsSuspendedFrame(Zone* zone,
                             const Function& function,
                             StackFrame* frame) {
  if (!function.IsSuspendableFunction()) {
    return false;
  }
  auto& suspend_state = Object::Handle(
      zone, *reinterpret_cast<ObjectPtr*>(LocalVarAddress(
                frame->fp(), runtime_frame_layout.FrameSlotForVariableIndex(
                                 SuspendState::kSuspendStateVarIndex))));
  return suspend_state.IsSuspendState() &&
         (SuspendState::Cast(suspend_state).pc() != 0);
}

void DeoptimizeAt(Thread* mutator_thread,
                  const Code& optimized_code,
                  StackFrame* frame) {
  ASSERT(optimized_code.can_be_deoptimized());

  Thread* thread = Thread::Current();
  Zone* zone = thread->zone();
  const Function& function = Function::Handle(zone, optimized_code.function());
  const Error& error =
      Error::Handle(zone, Compiler::EnsureUnoptimizedCode(thread, function));
  if (!error.IsNull()) {
    Exceptions::PropagateError(error);
  }
  const Code& unoptimized_code =
      Code::Handle(zone, function.unoptimized_code());
  ASSERT(!unoptimized_code.IsNull());
  // The switch to unoptimized code may have already occurred.
  if (function.HasOptimizedCode()) {
    function.SwitchToUnoptimizedCode();
  }

  if (IsSuspendedFrame(zone, function, frame)) {
    // Frame is suspended and going to be removed from the stack.
    if (FLAG_trace_deoptimization) {
      THR_Print("Not deoptimizing suspended frame, fp=%" Pp "\n", frame->fp());
    }
  } else if (frame->IsMarkedForLazyDeopt()) {
    // Deopt already scheduled.
    if (FLAG_trace_deoptimization) {
      THR_Print("Lazy deopt already scheduled for fp=%" Pp "\n", frame->fp());
    }
  } else {
    uword deopt_pc = frame->pc();
    ASSERT(optimized_code.ContainsInstructionAt(deopt_pc));

#if defined(DEBUG)
    ValidateFrames();
#endif

    // N.B.: Update the pending deopt table before updating the frame. The
    // profiler may attempt a stack walk in between.
    ASSERT(!frame->is_interpreted());
    mutator_thread->pending_deopts().AddPendingDeopt(frame->fp(), deopt_pc);
    frame->MarkForLazyDeopt();

    if (FLAG_trace_deoptimization) {
      THR_Print("Lazy deopt scheduled for fp=%" Pp ", pc=%" Pp "\n",
                frame->fp(), deopt_pc);
    }
  }

  // Mark code as dead (do not GC its embedded objects).
  optimized_code.set_is_alive(false);
}

// Currently checks only that all optimized frames have kDeoptIndex
// and unoptimized code has the kDeoptAfter.
void DeoptimizeFunctionsOnStack() {
  auto thread = Thread::Current();
  // Have to grab program_lock before stopping everybody else.
  SafepointWriteRwLocker ml(thread, thread->isolate_group()->program_lock());

  auto isolate_group = thread->isolate_group();
  isolate_group->RunWithStoppedMutators([&]() {
    Code& optimized_code = Code::Handle();
    isolate_group->ForEachIsolate(
        [&](Isolate* isolate) {
          auto mutator_thread = isolate->mutator_thread();
          if (mutator_thread == nullptr) {
            return;
          }
          DartFrameIterator iterator(
              mutator_thread, StackFrameIterator::kAllowCrossThreadIteration);
          StackFrame* frame = iterator.NextFrame();
          while (frame != nullptr) {
            if (!frame->is_interpreted()) {
              optimized_code = frame->LookupDartCode();
              // TODO(61635): handle code generated by modular AOT.
              if (optimized_code.can_be_deoptimized()) {
                DeoptimizeAt(mutator_thread, optimized_code, frame);
              }
            }
            frame = iterator.NextFrame();
          }
        },
        /*at_safepoint=*/true);
  });
}

static void DeoptimizeLastDartFrameIfOptimized() {
  auto thread = Thread::Current();
  // Have to grab program_lock before stopping everybody else.
  SafepointWriteRwLocker ml(thread, thread->isolate_group()->program_lock());

  auto isolate = thread->isolate();
  auto isolate_group = thread->isolate_group();
  isolate_group->RunWithStoppedMutators([&]() {
    auto mutator_thread = isolate->mutator_thread();
    if (mutator_thread == nullptr) {
      return;
    }
    DartFrameIterator iterator(mutator_thread,
                               StackFrameIterator::kNoCrossThreadIteration);
    StackFrame* frame = iterator.NextFrame();
    if (frame != nullptr && !frame->is_interpreted()) {
      const auto& optimized_code = Code::Handle(frame->LookupDartCode());
      // TODO(61635): handle code generated by modular AOT.
      if (optimized_code.can_be_deoptimized()) {
        DeoptimizeAt(mutator_thread, optimized_code, frame);
      }
    }
  });
}

#if !defined(DART_PRECOMPILED_RUNTIME)
static constexpr intptr_t kNumberOfSavedCpuRegisters = kNumberOfCpuRegisters;
static constexpr intptr_t kNumberOfSavedFpuRegisters = kNumberOfFpuRegisters;

static void CopySavedRegisters(uword saved_registers_address,
                               fpu_register_t** fpu_registers,
                               intptr_t** cpu_registers) {
  // Tell MemorySanitizer this region is initialized by generated code. This
  // region isn't already (fully) unpoisoned by FrameSetIterator::Unpoison
  // because it is in an exit frame and stack frame iteration doesn't have
  // access to true SP for exit frames.
  MSAN_UNPOISON(reinterpret_cast<void*>(saved_registers_address),
                kNumberOfSavedFpuRegisters * kFpuRegisterSize +
                    kNumberOfSavedCpuRegisters * kWordSize);

  ASSERT(sizeof(fpu_register_t) == kFpuRegisterSize);
  fpu_register_t* fpu_registers_copy =
      new fpu_register_t[kNumberOfSavedFpuRegisters];
  ASSERT(fpu_registers_copy != nullptr);
  for (intptr_t i = 0; i < kNumberOfSavedFpuRegisters; i++) {
    fpu_registers_copy[i] =
        *reinterpret_cast<fpu_register_t*>(saved_registers_address);
    saved_registers_address += kFpuRegisterSize;
  }
  *fpu_registers = fpu_registers_copy;

  ASSERT(sizeof(intptr_t) == kWordSize);
  intptr_t* cpu_registers_copy = new intptr_t[kNumberOfSavedCpuRegisters];
  ASSERT(cpu_registers_copy != nullptr);
  for (intptr_t i = 0; i < kNumberOfSavedCpuRegisters; i++) {
    cpu_registers_copy[i] =
        *reinterpret_cast<intptr_t*>(saved_registers_address);
    saved_registers_address += kWordSize;
  }
  *cpu_registers = cpu_registers_copy;
}
#endif

extern "C" bool DLRT_TryDoubleAsInteger(Thread* thread) {
  double value = thread->unboxed_double_runtime_arg();
  int64_t int_value = static_cast<int64_t>(value);
  double converted_double = static_cast<double>(int_value);
  if (converted_double != value) {
    return false;
  }
  thread->set_unboxed_int64_runtime_arg(int_value);
  return true;
}
DEFINE_LEAF_RUNTIME_ENTRY(TryDoubleAsInteger, 1, DLRT_TryDoubleAsInteger);

// Copies saved registers and caller's frame into temporary buffers.
// Returns the stack size of unoptimized frame.
// The calling code must be optimized, but its function may not have
// have optimized code if the code is OSR code, or if the code was invalidated
// through class loading/finalization or field guard.
extern "C" intptr_t DLRT_DeoptimizeCopyFrame(uword saved_registers_address,
                                             uword is_lazy_deopt) {
#if !defined(DART_PRECOMPILED_RUNTIME)
  Thread* thread = Thread::Current();
  StackZone zone(thread);

  // All registers have been saved below last-fp as if they were locals.
  const uword last_fp =
      saved_registers_address + (kNumberOfSavedCpuRegisters * kWordSize) +
      (kNumberOfSavedFpuRegisters * kFpuRegisterSize) -
      ((runtime_frame_layout.first_local_from_fp + 1) * kWordSize);

  // Get optimized code and frame that need to be deoptimized.
  DartFrameIterator iterator(last_fp, thread,
                             StackFrameIterator::kNoCrossThreadIteration);

  StackFrame* caller_frame = iterator.NextFrame();
  ASSERT(caller_frame != nullptr);
  const Code& optimized_code = Code::Handle(caller_frame->LookupDartCode());
  ASSERT(optimized_code.is_optimized());
  const Function& top_function =
      Function::Handle(thread->zone(), optimized_code.function());
  const bool deoptimizing_code = top_function.HasOptimizedCode();
  if (FLAG_trace_deoptimization) {
    const Function& function = Function::Handle(optimized_code.function());
    THR_Print("== Deoptimizing code for '%s', %s, %s\n",
              function.ToFullyQualifiedCString(),
              deoptimizing_code ? "code & frame" : "frame",
              (is_lazy_deopt != 0u) ? "lazy-deopt" : "");
  }

  if (is_lazy_deopt != 0u) {
    const uword deopt_pc =
        thread->pending_deopts().FindPendingDeopt(caller_frame->fp());

    // N.B.: Update frame before updating pending deopt table. The profiler
    // may attempt a stack walk in between.
    caller_frame->set_pc(deopt_pc);
    ASSERT(caller_frame->pc() == deopt_pc);
    ASSERT(optimized_code.ContainsInstructionAt(caller_frame->pc()));
    thread->pending_deopts().ClearPendingDeoptsAtOrBelow(
        caller_frame->fp(), PendingDeopts::kClearDueToDeopt);
  } else {
    if (FLAG_trace_deoptimization) {
      THR_Print("Eager deopt fp=%" Pp " pc=%" Pp "\n", caller_frame->fp(),
                caller_frame->pc());
    }
  }

  // Copy the saved registers from the stack.
  fpu_register_t* fpu_registers;
  intptr_t* cpu_registers;
  CopySavedRegisters(saved_registers_address, &fpu_registers, &cpu_registers);

  // Create the DeoptContext.
  DeoptContext* deopt_context = new DeoptContext(
      caller_frame, optimized_code, DeoptContext::kDestIsOriginalFrame,
      fpu_registers, cpu_registers, is_lazy_deopt != 0, deoptimizing_code);
  thread->set_deopt_context(deopt_context);

  // Stack size (FP - SP) in bytes.
  return deopt_context->DestStackAdjustment() * kWordSize;
#else
  UNREACHABLE();
  return 0;
#endif  // !DART_PRECOMPILED_RUNTIME
}
DEFINE_LEAF_RUNTIME_ENTRY(DeoptimizeCopyFrame, 2, DLRT_DeoptimizeCopyFrame);

// The stack has been adjusted to fit all values for unoptimized frame.
// Fill the unoptimized frame.
extern "C" intptr_t DLRT_DeoptimizeFillFrame(uword last_fp) {
#if !defined(DART_PRECOMPILED_RUNTIME)
  Thread* thread = Thread::Current();
  StackZone zone(thread);

  DeoptContext* deopt_context = thread->deopt_context();
  DartFrameIterator iterator(last_fp, thread,
                             StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* caller_frame = iterator.NextFrame();
  ASSERT(caller_frame != nullptr);

#if defined(DEBUG)
  {
    // The code from the deopt_context.
    const Code& code = Code::Handle(deopt_context->code());

    // The code from our frame.
    const Code& optimized_code = Code::Handle(caller_frame->LookupDartCode());
    const Function& function = Function::Handle(optimized_code.function());
    ASSERT(!function.IsNull());

    // The code will be the same as before.
    ASSERT(code.ptr() == optimized_code.ptr());

    // Some sanity checking of the optimized code.
    ASSERT(!optimized_code.IsNull() && optimized_code.is_optimized());
  }
#endif

  deopt_context->set_dest_frame(caller_frame);
  intptr_t frame_count = deopt_context->FillDestFrame();
  ASSERT(frame_count > 0);
  if (FLAG_trace_deoptimization) {
    THR_Print("Deopt created %" Pd " frames\n", frame_count);
  }
  return frame_count;
#else
  UNREACHABLE();
  return 0;
#endif  // !DART_PRECOMPILED_RUNTIME
}
DEFINE_LEAF_RUNTIME_ENTRY(DeoptimizeFillFrame, 1, DLRT_DeoptimizeFillFrame);

// This is the last step in the deoptimization, GC can occur.
// Returns number of bytes to remove from the expression stack of the
// bottom-most deoptimized frame. Those arguments were artificially injected
// under return address to keep them discoverable by GC that can occur during
// materialization phase.
DEFINE_RUNTIME_ENTRY(DeoptimizeMaterialize, 0) {
#if !defined(DART_PRECOMPILED_RUNTIME)
#if defined(DEBUG)
  {
    // We may rendezvous for a safepoint at entry or GC from the allocations
    // below. Check the stack is walkable.
    ValidateFrames();
  }
#endif
  DeoptContext* deopt_context = thread->deopt_context();
  intptr_t deopt_arg_count = deopt_context->MaterializeDeferredObjects();
  thread->set_deopt_context(nullptr);
  delete deopt_context;

  // Return value tells deoptimization stub to remove the given number of bytes
  // from the stack.
  arguments.SetReturn(Smi::Handle(Smi::New(deopt_arg_count * kWordSize)));
#else
  UNREACHABLE();
#endif  // !DART_PRECOMPILED_RUNTIME
}

DEFINE_RUNTIME_ENTRY(RewindPostDeopt, 0) {
#if !defined(DART_PRECOMPILED_RUNTIME)
#if !defined(PRODUCT)
  isolate->debugger()->RewindPostDeopt();
#endif  // !PRODUCT
#endif  // !DART_PRECOMPILED_RUNTIME
  UNREACHABLE();
}

// Handle slow path actions for the resumed frame after it was
// copied back to the stack:
// 1) deoptimization;
// 2) breakpoint at resumption;
// 3) throwing an exception.
//
// Arg0: exception
// Arg1: stack trace
DEFINE_RUNTIME_ENTRY(ResumeFrame, 2) {
  const Instance& exception = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const Instance& stacktrace =
      Instance::CheckedHandle(zone, arguments.ArgAt(1));

#if !defined(DART_PRECOMPILED_RUNTIME)
#if !defined(PRODUCT)
  if (isolate != nullptr) {
    if (isolate->has_resumption_breakpoints()) {
      isolate->debugger()->ResumptionBreakpoint();
    }
  }
#endif

  DartFrameIterator iterator(thread,
                             StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* frame = iterator.NextFrame();
  ASSERT(frame->IsDartFrame());
  ASSERT(!frame->is_interpreted());
  ASSERT(Function::Handle(zone, frame->LookupDartFunction())
             .IsSuspendableFunction());
  const Code& caller_code = Code::Handle(zone, frame->LookupDartCode());
  if (caller_code.IsDisabled() && caller_code.can_be_deoptimized()) {
    const uword deopt_pc = frame->pc();
    thread->pending_deopts().AddPendingDeopt(frame->fp(), deopt_pc);
    frame->MarkForLazyDeopt();

    if (FLAG_trace_deoptimization) {
      THR_Print("Lazy deopt scheduled for resumed frame fp=%" Pp ", pc=%" Pp
                "\n",
                frame->fp(), deopt_pc);
    }
  }
#endif

  if (!exception.IsNull()) {
    Exceptions::ReThrow(thread, exception, stacktrace);
  }
}

void OnEveryRuntimeEntryCall(Thread* thread,
                             const char* runtime_call_name,
                             bool can_lazy_deopt) {
  ASSERT(FLAG_deoptimize_on_runtime_call_every > 0);
  if (FLAG_precompiled_mode) {
    return;
  }
  if (IsolateGroup::IsSystemIsolateGroup(thread->isolate_group())) {
    return;
  }
  const bool is_deopt_related =
      strstr(runtime_call_name, "Deoptimize") != nullptr;
  if (is_deopt_related) {
    return;
  }
  // For --deoptimize-on-every-runtime-call we only consider runtime calls that
  // can lazy-deopt.
  if (can_lazy_deopt) {
    if (FLAG_deoptimize_on_runtime_call_name_filter != nullptr &&
        (strlen(runtime_call_name) !=
             strlen(FLAG_deoptimize_on_runtime_call_name_filter) ||
         strstr(runtime_call_name,
                FLAG_deoptimize_on_runtime_call_name_filter) == nullptr)) {
      return;
    }
    const uint32_t count = thread->IncrementAndGetRuntimeCallCount();
    if ((count % FLAG_deoptimize_on_runtime_call_every) == 0) {
      DeoptimizeLastDartFrameIfOptimized();
    }
  }
}

double DartModulo(double left, double right) {
  double remainder = fmod_ieee(left, right);
  if (remainder == 0.0) {
    // We explicitly switch to the positive 0.0 (just in case it was negative).
    remainder = +0.0;
  } else if (remainder < 0.0) {
    if (right < 0) {
      remainder -= right;
    } else {
      remainder += right;
    }
  }
  return remainder;
}

// Update global type feedback recorded for a field recording the assignment
// of the given value.
//   Arg0: Field object;
//   Arg1: Value that is being stored.
DEFINE_RUNTIME_ENTRY(UpdateFieldCid, 2) {
#if !defined(DART_PRECOMPILED_RUNTIME)
  const Field& field = Field::CheckedHandle(zone, arguments.ArgAt(0));
  const Object& value = Object::Handle(arguments.ArgAt(1));
  field.RecordStore(value);
#else
  UNREACHABLE();
#endif
}

DEFINE_RUNTIME_ENTRY(InitInstanceField, 2) {
  const Instance& instance = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const Field& field = Field::CheckedHandle(zone, arguments.ArgAt(1));
  Object& result = Object::Handle(zone, field.InitializeInstance(instance));
  ThrowIfError(result);
  result = instance.GetField(field);
  ASSERT(result.ptr() != Object::sentinel().ptr());
  arguments.SetReturn(result);
}

DEFINE_RUNTIME_ENTRY(InitStaticField, 1) {
  const Field& field = Field::CheckedHandle(zone, arguments.ArgAt(0));
  Object& result = Object::Handle(zone, field.InitializeStatic());
  ThrowIfError(result);
  result = field.StaticValue();
  ASSERT(result.ptr() != Object::sentinel().ptr());
  arguments.SetReturn(result);
}

DEFINE_RUNTIME_ENTRY(CheckedStoreIntoShared, 2) {
  const Field& field = Field::CheckedHandle(zone, arguments.ArgAt(0));
  const Instance& value = Instance::CheckedHandle(zone, arguments.ArgAt(1));

  value.EnsureDeeplyImmutable(zone);

  field.SetStaticValue(value);
  arguments.SetReturn(field);
}

DEFINE_RUNTIME_ENTRY(StaticFieldAccessedWithoutIsolateError, 1) {
  const Field& field = Field::CheckedHandle(zone, arguments.ArgAt(0));
  Exceptions::ThrowStaticFieldAccessedWithoutIsolate(
      String::Handle(field.name()));
  UNREACHABLE();
}

DEFINE_RUNTIME_ENTRY(LateFieldAlreadyInitializedError, 1) {
  const Field& field = Field::CheckedHandle(zone, arguments.ArgAt(0));
  Exceptions::ThrowLateFieldAlreadyInitialized(String::Handle(field.name()));
}

DEFINE_RUNTIME_ENTRY(LateFieldAssignedDuringInitializationError, 1) {
  const Field& field = Field::CheckedHandle(zone, arguments.ArgAt(0));
  Exceptions::ThrowLateFieldAssignedDuringInitialization(
      String::Handle(field.name()));
}

DEFINE_RUNTIME_ENTRY(LateFieldNotInitializedError, 1) {
  const Field& field = Field::CheckedHandle(zone, arguments.ArgAt(0));
  Exceptions::ThrowLateFieldNotInitialized(String::Handle(field.name()));
}

DEFINE_RUNTIME_ENTRY(NotLoaded, 0) {
  // We could just use a trap instruction in the stub, but we get better stack
  // traces when there is an exit frame.
  FATAL("Not loaded");
}

DEFINE_RUNTIME_ENTRY(FfiAsyncCallbackSend, 1) {
  Dart_Port target_port = thread->unboxed_int64_runtime_arg();
  TRACE_RUNTIME_CALL("FfiAsyncCallbackSend %p", (void*)target_port);
  const Object& message = Object::Handle(zone, arguments.ArgAt(0));
  const Array& msg_array = Array::Handle(zone, Array::New(3));
  msg_array.SetAt(0, message);
  PersistentHandle* handle =
      thread->isolate_group()->api_state()->AllocatePersistentHandle();
  handle->set_ptr(msg_array);
  PortMap::PostMessage(
      Message::New(target_port, handle, Message::kNormalPriority));
}

// Use expected function signatures to help MSVC compiler resolve overloading.
typedef double (*UnaryMathCFunction)(double x);
typedef double (*BinaryMathCFunction)(double x, double y);
typedef void* (*MemMoveCFunction)(void* dest, const void* src, size_t n);

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcPow,
                                /*argument_count=*/2,
                                static_cast<BinaryMathCFunction>(pow));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(DartModulo,
                                /*argument_count=*/2,
                                static_cast<BinaryMathCFunction>(DartModulo));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcFmod,
                                /*argument_count=*/2,
                                static_cast<BinaryMathCFunction>(fmod_ieee));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcAtan2,
                                /*argument_count=*/2,
                                static_cast<BinaryMathCFunction>(atan2_ieee));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcFloor,
                                /*argument_count=*/1,
                                static_cast<UnaryMathCFunction>(floor));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcCeil,
                                /*argument_count=*/1,
                                static_cast<UnaryMathCFunction>(ceil));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcTrunc,
                                /*argument_count=*/1,
                                static_cast<UnaryMathCFunction>(trunc));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcRound,
                                /*argument_count=*/1,
                                static_cast<UnaryMathCFunction>(round));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcCos,
                                /*argument_count=*/1,
                                static_cast<UnaryMathCFunction>(cos));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcSin,
                                /*argument_count=*/1,
                                static_cast<UnaryMathCFunction>(sin));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcAsin,
                                /*argument_count=*/1,
                                static_cast<UnaryMathCFunction>(asin));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcAcos,
                                /*argument_count=*/1,
                                static_cast<UnaryMathCFunction>(acos));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcTan,
                                /*argument_count=*/1,
                                static_cast<UnaryMathCFunction>(tan));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcAtan,
                                /*argument_count=*/1,
                                static_cast<UnaryMathCFunction>(atan));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcExp,
                                /*argument_count=*/1,
                                static_cast<UnaryMathCFunction>(exp));

DEFINE_FLOAT_LEAF_RUNTIME_ENTRY(LibcLog,
                                /*argument_count=*/1,
                                static_cast<UnaryMathCFunction>(log));

DEFINE_LEAF_RUNTIME_ENTRY(MemoryMove,
                          /*argument_count=*/3,
                          static_cast<MemMoveCFunction>(memmove));

#if defined(DART_DYNAMIC_MODULES)
// Interpret a function call. Should be called only for non-jitted functions.
// argc indicates the number of arguments, including the type arguments.
// argv points to the first argument.
// If argc < 0, arguments are passed at decreasing memory addresses from argv.
extern "C" uword /*ObjectPtr*/ InterpretCall(uword /*FunctionPtr*/ function_in,
                                             uword /*ArrayPtr*/ argdesc_in,
                                             intptr_t argc,
                                             ObjectPtr* argv,
                                             Thread* thread) {
  FunctionPtr function = static_cast<FunctionPtr>(function_in);
  ArrayPtr argdesc = static_cast<ArrayPtr>(argdesc_in);
  Interpreter* interpreter = Interpreter::Current();
#if defined(DEBUG)
  uword exit_fp = thread->top_exit_frame_info();
  ASSERT(exit_fp != 0);
  ASSERT(thread == Thread::Current());
  // Caller is InterpretCall stub called from generated code.
  // We stay in "in generated code" execution state when interpreting code.
  ASSERT(thread->execution_state() == Thread::kThreadInGenerated);
  ASSERT(Function::HasBytecode(function));
  ASSERT(Function::IsInterpreted(function));
  ASSERT(interpreter != nullptr);
#endif
  // Tell MemorySanitizer 'argv' is initialized by generated code.
  if (argc < 0) {
    MSAN_UNPOISON(argv - argc, -argc * sizeof(ObjectPtr));
  } else {
    MSAN_UNPOISON(argv, argc * sizeof(ObjectPtr));
  }
  ObjectPtr result =
      interpreter->Call(function, argdesc, argc, argv, Array::null(), thread);
  DEBUG_ASSERT(thread->top_exit_frame_info() == exit_fp);
  if (IsErrorClassId(result->GetClassId())) [[unlikely]] {
    // Since there may not be an active zone (e.g., an isolate group bound
    // callback), make one. This also ensures that any handles allocated due
    // to things like debugging prints or throwing exceptions are not leaked
    // into the caller's zone (when present).
    StackZone stack_zone(thread);
    // Protect the result in a handle before transitioning, which may trigger
    // GC.
    Zone* const zone = stack_zone.GetZone();
    const Error& error = Error::Handle(zone, Error::RawCast(result));
    // Propagating an error may cause allocation. Check if we need to block for
    // a safepoint by switching to "in VM" execution state.
    TransitionGeneratedToVM transition(thread);
    PRINT_IF_TRACING_INTERPRETER("throwing exception: %s\n",
                                 error.ToErrorCString());
    Exceptions::PropagateError(error);
  }
  return static_cast<uword>(result);
}
#endif  // defined(DART_DYNAMIC_MODULES)

uword RuntimeEntry::InterpretCallEntry() {
#if defined(DART_DYNAMIC_MODULES)
  uword entry = reinterpret_cast<uword>(InterpretCall);
#if defined(DART_INCLUDE_SIMULATOR)
  if (FLAG_use_simulator) {
    entry = Simulator::RedirectExternalReference(
        entry, Simulator::kLeafRuntimeCall, 5);
  }
#endif
  return entry;
#else
  return 0;
#endif  // defined(DART_DYNAMIC_MODULES)
}

// Restore suspended interpreter frame and resume execution.
//
// Arg0: return value (result of the suspension)
// Arg1: exception
// Arg2: stack trace
DEFINE_RUNTIME_ENTRY(ResumeInterpreter, 3) {
#if defined(DART_DYNAMIC_MODULES)
  const Instance& value = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  const Instance& exception = Instance::CheckedHandle(zone, arguments.ArgAt(1));
  const Instance& stack_trace =
      Instance::CheckedHandle(zone, arguments.ArgAt(2));

  StackFrameIterator iterator(ValidationPolicy::kDontValidateFrames, thread,
                              StackFrameIterator::kNoCrossThreadIteration);
  StackFrame* frame = iterator.NextFrame();
  ASSERT(frame != nullptr);
  while (frame->IsExitFrame() ||
         (frame->IsStubFrame() &&
          !StubCode::ResumeInterpreter().ContainsInstructionAt(frame->pc()) &&
          !StubCode::Resume().ContainsInstructionAt(frame->pc()))) {
    frame = iterator.NextFrame();
    ASSERT(frame != nullptr);
  }
  RELEASE_ASSERT(frame->IsStubFrame());

  const uword fp = frame->fp();
  const uword sp = arguments.GetCallerSP();
  ASSERT((fp > sp) && (sp > frame->sp()));
  MSAN_UNPOISON(reinterpret_cast<uint8_t*>(sp), fp - sp);

  Interpreter* interpreter = Interpreter::Current();
  auto& result = Object::Handle(zone);

  {
    TransitionVMToGenerated transition(thread);
    result = interpreter->Resume(thread, fp, sp, value.ptr(), exception.ptr(),
                                 stack_trace.ptr());
  }
  if (IsErrorClassId(result.GetClassId())) [[unlikely]] {
    Exceptions::PropagateError(Error::Cast(result));
  }
  arguments.SetReturn(result);
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES)
}

// Lazily allocates a coverage array for bytecode prior to recording coverage.
//
// Arg0: Bytecode object that needs an allocated coverage array.
DEFINE_RUNTIME_ENTRY(AllocateBytecodeCoverageArray, 1) {
#if defined(DART_DYNAMIC_MODULES) && !defined(PRODUCT) &&                      \
    !defined(DART_PRECOMPILED_RUNTIME)
  const auto& bytecode = Bytecode::CheckedHandle(zone, arguments.ArgAt(0));
  const auto& coverage_array =
      TypedData::Handle(zone, bytecode.EnsureCoverageArray(thread));
  arguments.SetReturn(coverage_array);
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES) && !defined(PRODUCT) &&
        // !defined(DART_PRECOMPILED_RUNTIME)
}

#if defined(DART_DYNAMIC_MODULES) && !defined(PRODUCT) &&                      \
    !defined(DART_PRECOMPILED_RUNTIME)
static ArrayPtr PrepareNoSuchMethodErrorArguments(Thread* thread,
                                                  Zone* zone,
                                                  const Function& target,
                                                  const Array& argdesc) {
  auto const kind = InvocationMirror::Kind::kMethod;
  const auto& owner = Class::Handle(target.Owner());
  auto& receiver = Instance::Handle();
  auto level = InvocationMirror::Level::kDynamic;
  if (owner.IsTopLevel()) {
    level = InvocationMirror::Level::kTopLevel;
  } else if (target.IsConstructor()) {
    level = InvocationMirror::Level::kConstructor;
  } else if (target.is_static()) {
    level = InvocationMirror::Level::kStatic;
  }
  const auto& member_name = String::Handle(zone, target.name());
  const auto& invocation_type =
      Smi::Handle(zone, Smi::New(InvocationMirror::EncodeType(level, kind)));

  // Collect the arguments to report for the NSM error by retrieving
  // the arguments to the old implicit closure from the stack.
  DartFrameIterator iterator(thread,
                             StackFrameIterator::kNoCrossThreadIteration);
  auto* const caller_frame = iterator.NextFrame();
  ASSERT(caller_frame->is_interpreted());

  ArgumentsDescriptor args_desc(argdesc);
  const intptr_t type_args_len = args_desc.TypeArgsLen();
  const intptr_t argc = args_desc.CountWithTypeArgs();
  const intptr_t receiver_idx = args_desc.FirstArgIndex();
  // The reported arguments don't include either the type arguments or
  // the receiver, as those are passed separately.
  const bool is_instance = !target.IsConstructor() && !target.is_static();
  const intptr_t arg_count =
      argc - (type_args_len > 0 ? 1 : 0) - (is_instance ? 1 : 0);
  const auto& orig_arguments = Array::Handle(zone, Array::New(arg_count));
  auto& obj = Object::Handle(zone);
  auto* const argv = reinterpret_cast<ObjectPtr*>(caller_frame->fp()) -
                     (kKBCDartFrameFixedSize + argc);
  auto& type_args = TypeArguments::Handle(zone);
  if (type_args_len > 0) {
    type_args ^= argv[0];
  }
  if (is_instance) {
    // The "context" of an implicit instance closure is the receiver.
    const auto& closure = Closure::CheckedHandle(zone, argv[receiver_idx]);
    receiver ^= closure.RawContext();
  } else {
    // The "receiver" of the NSM error is the class (represented as a type).
    receiver = owner.RareType();
  }
  for (intptr_t i = 0; i < arg_count; i++) {
    obj = argv[receiver_idx + i];
    orig_arguments.SetAt(i, obj);
  }

  // NoSuchMethodError._throwNew takes the following arguments:
  //   Object receiver,
  //   String memberName,
  //   int invocationType,
  //   int typeArgumentsLength,
  //   Object? typeArguments,
  //   List? arguments,
  //   List? argumentNames
  const auto& args = Array::Handle(zone, Array::New(7));
  args.SetAt(0, receiver);
  args.SetAt(1, member_name);
  args.SetAt(2, invocation_type);
  args.SetAt(3, Smi::Handle(zone, Smi::New(type_args_len)));
  args.SetAt(4, type_args);
  args.SetAt(5, orig_arguments);
  args.SetAt(6, Object::null_object());
  return args.ptr();
}
#endif  // defined(DART_DYNAMIC_MODULES) && !defined(PRODUCT) &&
        // !defined(DART_PRECOMPILED_RUNTIME)

// Re-resolves the function for an interpreted implicit closure after reload.
// Either returns the new implicit closure function or an array of arguments
// to pass to NoSuchMethodError._throwNew.
//
// Arg0: Implicit closure function that should be re-resolved.
// Arg1: Arguments descriptor array for the call.
DEFINE_RUNTIME_ENTRY(ResolveReloadedImplicitClosureFunction, 2) {
#if defined(DART_DYNAMIC_MODULES) && !defined(PRODUCT) &&                      \
    !defined(DART_PRECOMPILED_RUNTIME)
  const auto& function = Function::CheckedHandle(zone, arguments.ArgAt(0));
  const auto& argdesc = Array::CheckedHandle(zone, arguments.ArgAt(1));
  ASSERT(function.IsImplicitClosureFunction());
  ASSERT(function.IsReloadedImplicitClosure());

  // The owner of the old implicit closure is the new class (via a PatchClass).
  const auto& klass = Class::Handle(zone, function.Owner());
  const auto& selector = String::Handle(zone, function.name());
  auto& result =
      Object::Handle(zone, Resolver::ResolveFunction(zone, klass, selector));
  if (!result.IsNull()) {
    // Get the new implicit closure function for the given target.
    result = Function::Cast(result).ImplicitClosureFunction();
    ASSERT(!Function::Cast(result).IsReloadedImplicitClosure());
    ArgumentsDescriptor args_desc(argdesc);
    if (!Function::Cast(result).AreValidArguments(args_desc,
                                                  /*error_message=*/nullptr)) {
      // Discard the resolved function and throw NSM instead as the arguments
      // have changed in an incompatible way.
      result = Function::null();
    }
  }
  if (result.IsNull()) {
    const auto& target = Function::Handle(zone, function.parent_function());
    result = PrepareNoSuchMethodErrorArguments(thread, zone, target, argdesc);
  }
  ASSERT(!result.IsNull());
  arguments.SetReturn(result);
#else
  UNREACHABLE();
#endif  // defined(DART_DYNAMIC_MODULES) && !defined(PRODUCT) &&
        // !defined(DART_PRECOMPILED_RUNTIME)
}

DEFINE_RUNTIME_ENTRY(FatalError, 1) {
  const String& message = String::CheckedHandle(zone, arguments.ArgAt(0));
  FATAL("%s", message.ToCString());
}

extern "C" void DLRT_EnterSafepoint() {
  CHECK_STACK_ALIGNMENT;
  TRACE_RUNTIME_CALL("%s", "EnterSafepoint");
  Thread* thread = Thread::Current();
  ASSERT(thread->top_exit_frame_info() != 0);
  ASSERT(thread->execution_state() == Thread::kThreadInNative);
  thread->EnterSafepointToNative();
  TRACE_RUNTIME_CALL("%s", "EnterSafepoint done");
}
DEFINE_LEAF_RUNTIME_ENTRY(EnterSafepoint,
                          /*argument_count=*/0,
                          DLRT_EnterSafepoint);

extern "C" void DLRT_ExitSafepoint() {
  CHECK_STACK_ALIGNMENT;
  Thread* thread = Thread::Current();
  TRACE_RUNTIME_CALL("ExitSafepoint thread %p", thread);
  ASSERT(thread->top_exit_frame_info() != 0);

  if (thread->is_unwind_in_progress()) {
    // Clean up safepoint unwind error marker to prevent safepoint tripping.
    // The safepoint marker will get restored just before jumping back
    // to generated code.
    TransitionToVM transition(thread);
    thread->SetUnwindErrorInProgress(false);
    NoSafepointScope no_safepoint;
    Exceptions::PropagateError(Object::unwind_error());
  }
  if (thread->execution_state() == Thread::kThreadInNative) {
    thread->ExitSafepointFromNative();
  } else {
    ASSERT(thread->execution_state() == Thread::kThreadInVM);
    thread->ExitSafepoint();
  }

  TRACE_RUNTIME_CALL("%s", "ExitSafepoint done");
}
DEFINE_LEAF_RUNTIME_ENTRY(ExitSafepoint,
                          /*argument_count=*/0,
                          DLRT_ExitSafepoint);

namespace {
Thread* HandleAsyncFfiCallback(FfiCallbackMetadata::Metadata metadata) {
  // NOTE: This is only thread safe if the user is using the API correctly.
  // Otherwise, the callback could have been deleted and replaced, in which case
  // IsLive would still be true. Or it could have been deleted after we looked
  // it up, and the target isolate could be shut down. We delay recycling
  // callbacks as long as possible, so this check is better than nothing, but
  // it's not infallible. Ultimately it's the user's responsibility to avoid use
  // after free errors. Trying to lock FfiCallbackMetadata::lock_, or any
  // similar lock, leads to deadlocks.

  Isolate* target_isolate = metadata.target_isolate();

  Isolate* current_isolate = nullptr;
  Thread* current_thread = Thread::Current();
  if (current_thread != nullptr) {
    current_isolate = current_thread->isolate();
    if (current_thread->execution_state() != Thread::kThreadInNative) {
      FATAL("Cannot invoke native callback from a leaf call.");
    }
    current_thread->ExitSafepointFromNative();
    current_thread->set_execution_state(Thread::kThreadInVM);
  }

  // Enter the temporary isolate. If the current isolate is in the same group
  // as the target isolate, we can skip entering the temp isolate, and marshal
  // the args on the current isolate.
  if (current_isolate == nullptr ||
      current_isolate->group() != target_isolate->group()) {
    if (current_isolate != nullptr) {
      Thread::ExitIsolate(/*isolate_shutdown=*/false);
    }
    target_isolate->group()->EnterTemporaryIsolate();
  }
  Thread* const temp_thread = Thread::Current();
  ASSERT(temp_thread != nullptr);
  temp_thread->set_unboxed_int64_runtime_arg(metadata.send_port());
  temp_thread->set_unboxed_int64_runtime_second_arg(
      reinterpret_cast<intptr_t>(current_isolate));
  ASSERT(!temp_thread->IsAtSafepoint());
  return temp_thread;
}

Thread* HandleIsolateGroupBoundSyncFfiCallback(
    FfiCallbackMetadata::Metadata metadata,
    CallbackMetadata* out) {
  Thread* current_thread = Thread::Current();

  if (current_thread != nullptr) {
    current_thread->ExitSafepointFromNative();
    current_thread->set_execution_state(Thread::kThreadInVM);
  }

  Isolate* current_isolate =
      current_thread != nullptr ? current_thread->isolate() : nullptr;

  bool should_enter_group = true;
  if (current_thread != nullptr) {
    if (current_isolate != nullptr) {
      Thread::ExitIsolate(/*isolate_shutdown=*/false);
    } else {
      IsolateGroup* current_isolategroup = current_thread->isolate_group();
      if (current_isolategroup != nullptr) {
        if (current_isolategroup == metadata.target_isolate_group()) {
          should_enter_group = false;
        } else {
          Thread::ExitIsolateGroupAsMutator(/*bypass_safepoint=*/false);
        }
      }
    }
  }
  if (should_enter_group) {
    Thread::EnterIsolateGroupAsMutator(metadata.target_isolate_group(),
                                       /*bypass_safepoint=*/false);
  }

  auto new_thread = Thread::Current();
  new_thread->set_execution_state(Thread::kThreadInVM);
  // We need to return to original isolate or isolate group if we were in them.
  out->caller_isolate = reinterpret_cast<uword>(current_isolate);
  out->caller_isolate_group = reinterpret_cast<uword>(
      current_thread != nullptr ? current_thread->isolate_group() : nullptr);
  current_thread = new_thread;

  current_thread->set_unboxed_int64_runtime_arg(metadata.context());

  return current_thread;
}

void FfiCallbackThreadChecks(Thread* thread, Isolate* target_isolate) {
  if (thread->no_callback_scope_depth() != 0) {
    FATAL("Cannot invoke native callback when API callbacks are prohibited.");
  }
  if (thread->is_unwind_in_progress()) {
    FATAL("Cannot invoke native callback while unwind error propagates.");
  }
  if (!thread->IsDartMutatorThread()) {
    FATAL("Native callbacks must be invoked on the mutator thread.");
  }
  if (thread->isolate() != target_isolate) {
    FATAL("Cannot invoke native callback from a different isolate.");
  }
}

Thread* HandleIsolateBoundSyncFfiCallback(
    FfiCallbackMetadata::Metadata metadata,
    CallbackMetadata* out) {
  Thread* current_thread = Thread::Current();

  Isolate* target_isolate = metadata.target_isolate();
  if (current_thread == nullptr) {
    if (!PortMap::IsOwnedByCurrentThread(target_isolate->main_port())) {
      FATAL("Cannot invoke native callback outside an isolate.");
    }
    Thread::EnterIsolate(target_isolate);
    current_thread = Thread::Current();
    FfiCallbackThreadChecks(current_thread, target_isolate);
    out->epilogue =
        reinterpret_cast<uword>(&DLRT_ExitSyncCallbackTargetIsolate);
  } else {
    FfiCallbackThreadChecks(current_thread, target_isolate);
    if (current_thread->execution_state() != Thread::kThreadInNative) {
      FATAL("Cannot invoke native callback from a leaf call.");
    }
    current_thread->ExitSafepointFromNative();
    out->epilogue = reinterpret_cast<uword>(&DLRT_ExitSyncCallback);
  }

  current_thread->set_execution_state(Thread::kThreadInVM);
  current_thread->set_unboxed_int64_runtime_arg(metadata.context());

  return current_thread;
}
}  // namespace

// This is called by a native callback trampoline
// (see StubCodeCompiler::GenerateFfiCallbackTrampolineStub). Not registered as
// a runtime entry because we can't use Thread to look it up.
extern "C" Thread* DLRT_GetFfiCallbackMetadata(
    FfiCallbackMetadata::Trampoline trampoline,
    CallbackMetadata* out) {
  CHECK_STACK_ALIGNMENT;
  TRACE_RUNTIME_CALL("GetFfiCallbackMetadata %p",
                     reinterpret_cast<void*>(trampoline));
  ASSERT(out != nullptr);

  if (!Isolate::IsolateCreationEnabled()) {
    FATAL("GetFfiCallbackMetadata called after shutdown %p",
          reinterpret_cast<void*>(trampoline));
  }

  // NOTE: We access the metadata for `trampoline` without a lock. This is safe
  // because nobody will touch the metadata of the `trampoline` until it's
  // deleted and the `NativeCallable` API requires the isolate to keep the
  // trampoline (and therefore the metadata) alive until C code no longer
  // attempts to call it.
  //
  // If a user of the `NativeCallable` API violates this agreement, we may
  // have a use-after-free scenario here and therefore undefined behavior.
  // We make some best effort to `FATAL()` in obvious cases of undefined
  // behavior, but not all cases will be caught.
  auto metadata = FfiCallbackMetadata::Instance(trampoline)
                      ->LookupMetadataForTrampolineUnlocked(trampoline);

  if (!metadata.IsLive()) {
    FATAL("Callback invoked after it has been deleted.");
  }

  Thread* thread = nullptr;

  out->entry_point = metadata.target_entry_point();
  out->function_handle = reinterpret_cast<uword>(metadata.function_handle());
#if defined(HOST_ARCH_ARM64) && defined(DART_DYNAMIC_MODULES)
  out->interpreted_runtime_entry =
      reinterpret_cast<uword>(DLRT_DoInterpretedFfiCallback);
#endif

  switch (metadata.trampoline_type()) {
    default:
      out->type = CallbackMetadata::kCall;
      break;
    case FfiCallbackMetadata::TrampolineType::kAsync:
      out->type = CallbackMetadata::kTailCall;
      break;
#if defined(TARGET_ARCH_IA32)
    case FfiCallbackMetadata::TrampolineType::kSyncStackDelta4:
    case FfiCallbackMetadata::TrampolineType::kSyncIsolateGroupBoundStackDelta4:
      out->type = CallbackMetadata::kCallRet4;
      break;
#endif
  }

  if (metadata.trampoline_type() ==
      FfiCallbackMetadata::TrampolineType::kAsync) {
    thread = HandleAsyncFfiCallback(metadata);
    out->epilogue = reinterpret_cast<uword>(&DLRT_ExitTemporaryIsolate);
  } else if (metadata.is_isolate_group_bound()) {
    thread = HandleIsolateGroupBoundSyncFfiCallback(metadata, out);
    out->epilogue = reinterpret_cast<uword>(&DLRT_ExitIsolateGroupBoundIsolate);
  } else {
    thread = HandleIsolateBoundSyncFfiCallback(metadata, out);
  }

  TRACE_RUNTIME_CALL("GetFfiCallbackMetadata thread %p", thread);
  TRACE_RUNTIME_CALL("GetFfiCallbackMetadata entry_point %p",
                     reinterpret_cast<void*>(out->entry_point));
  return thread;
}

#if defined(USING_MEMORY_SANITIZER)
// There is a __msan_unpoison_param but no __msan_unpoison_retval. Trick MSAN
// into unpoisoning the largest value that an FFI callback might return. The
// actual result registers will be overridden in the FFI callback stub.
struct LargestReturn {
  float x, y, w, z;
};
extern "C" LargestReturn dart_msan_unpoison_retval() {
  return {0, 0, 0, 0};
}
#endif

extern "C" void* DLRT_ExitIsolateGroupBoundIsolate(
    Thread* thread,
    Isolate* caller_isolate,
    IsolateGroup* caller_isolate_group) {
  TRACE_RUNTIME_CALL("ExitIsolateGroupBoundIsolate%s", "");
  ASSERT(thread != nullptr);
  ASSERT(thread == Thread::Current());
  // Need to accommodate ExitIsolateGroupAsHelper assumptions.
  thread->set_execution_state(Thread::kThreadInVM);
  Thread::ExitIsolateGroupAsMutator(/*bypass_safepoint=*/false);
  if (caller_isolate != nullptr || caller_isolate_group != nullptr) {
    if (caller_isolate != nullptr) {
      // if we were called from an isolate, return to that isolate
      Thread::EnterIsolate(caller_isolate);
    } else {
      ASSERT(caller_isolate_group != nullptr);
      // if we were called from an isolate group, return to that instead.
      Thread::EnterIsolateGroupAsMutator(caller_isolate_group,
                                         /*bypass_safepoint=*/false,
                                         /*suspended_thread=*/thread);
    }
    Thread::Current()->set_execution_state(Thread::kThreadInNative);
    Thread::Current()->EnterSafepoint();
  }
#if defined(USING_MEMORY_SANITIZER)
  return reinterpret_cast<void*>(dart_msan_unpoison_retval);
#else
  return nullptr;
#endif
}

extern "C" void* DLRT_ExitSyncCallbackTargetIsolate(
    Thread* thread,
    Isolate* caller_isolate,
    IsolateGroup* caller_isolate_group) {
  TRACE_RUNTIME_CALL("ExitSyncCallbackTargetIsolate%s", "");
  ASSERT(thread != nullptr);
  ASSERT(thread == Thread::Current());
  thread->set_execution_state(Thread::kThreadInVM);
  Thread::ExitIsolate(/*isolate_shutdown=*/false);
#if defined(USING_MEMORY_SANITIZER)
  return reinterpret_cast<void*>(dart_msan_unpoison_retval);
#else
  return nullptr;
#endif
}

extern "C" void* DLRT_ExitSyncCallback(Thread* thread,
                                       Isolate* caller_isolate,
                                       IsolateGroup* caller_isolate_group) {
  ASSERT(thread != nullptr);
  ASSERT(thread == Thread::Current());

  thread->set_execution_state(Thread::kThreadInNative);
  thread->EnterSafepointToNative();

#if defined(USING_MEMORY_SANITIZER)
  return reinterpret_cast<void*>(dart_msan_unpoison_retval);
#else
  return nullptr;
#endif
}

#if defined(HOST_ARCH_IA32)
// A function with arguments isn't compatible with the tail-call because on IA32
// the arguments are on the stack and caller pops.
extern "C" void* DLRT_ExitTemporaryIsolate() {
  Thread* thread = Thread::Current();
#else
extern "C" void* DLRT_ExitTemporaryIsolate(Thread* thread) {
#endif
  TRACE_RUNTIME_CALL("ExitTemporaryIsolate%s", "");
  ASSERT(thread != nullptr);
  ASSERT(thread == Thread::Current());
  Isolate* source_isolate =
      reinterpret_cast<Isolate*>(thread->unboxed_int64_runtime_second_arg());

  // We're either inside a temp isolate, or inside the source_isolate.
  const bool inside_temp_isolate =
      source_isolate == nullptr || source_isolate != thread->isolate();
  if (inside_temp_isolate) {
    IsolateGroup::ExitTemporaryIsolate();
    if (source_isolate != nullptr) {
      TRACE_RUNTIME_CALL("ExitTemporaryIsolate re-entering source isolate %p",
                         source_isolate);
      Thread::EnterIsolate(source_isolate);
      Thread::Current()->set_execution_state(Thread::kThreadInNative);
      Thread::Current()->EnterSafepoint();
    }
  } else {
    thread->set_execution_state(Thread::kThreadInNative);
    thread->EnterSafepoint();
  }
  TRACE_RUNTIME_CALL("ExitTemporaryIsolate %s", "done");
  return nullptr;
}

extern "C" ApiLocalScope* DLRT_EnterHandleScope(Thread* thread) {
  CHECK_STACK_ALIGNMENT;
  TRACE_RUNTIME_CALL("EnterHandleScope %p", thread);
  thread->EnterApiScope();
  ApiLocalScope* return_value = thread->api_top_scope();
  TRACE_RUNTIME_CALL("EnterHandleScope returning %p", return_value);
  return return_value;
}
DEFINE_LEAF_RUNTIME_ENTRY(EnterHandleScope,
                          /*argument_count=*/1,
                          DLRT_EnterHandleScope);

extern "C" void DLRT_ExitHandleScope(Thread* thread) {
  CHECK_STACK_ALIGNMENT;
  TRACE_RUNTIME_CALL("ExitHandleScope %p", thread);
  thread->ExitApiScope();
  TRACE_RUNTIME_CALL("ExitHandleScope %s", "done");
}
DEFINE_LEAF_RUNTIME_ENTRY(ExitHandleScope,
                          /*argument_count=*/1,
                          DLRT_ExitHandleScope);

extern "C" LocalHandle* DLRT_AllocateHandle(ApiLocalScope* scope) {
  CHECK_STACK_ALIGNMENT;
  TRACE_RUNTIME_CALL("AllocateHandle %p", scope);
  LocalHandle* return_value = scope->local_handles()->AllocateHandle();
  // Don't return an uninitialised handle.
  return_value->set_ptr(Object::sentinel().ptr());
  TRACE_RUNTIME_CALL("AllocateHandle returning %p", return_value);
  return return_value;
}
DEFINE_LEAF_RUNTIME_ENTRY(AllocateHandle,
                          /*argument_count=*/1,
                          DLRT_AllocateHandle);

// Enables reusing `Dart_PropagateError` from `FfiCallInstr`.
// `Dart_PropagateError` requires the native state and transitions into the VM.
// So the flow is:
// - FfiCallInstr (slow path)
// - TransitionGeneratedToNative
// - DLRT_PropagateError (this)
// - Dart_PropagateError
// - TransitionNativeToVM
// - Throw
extern "C" void DLRT_PropagateError(Dart_Handle handle) {
  CHECK_STACK_ALIGNMENT;
  TRACE_RUNTIME_CALL("PropagateError %p", handle);
  ASSERT(Thread::Current()->execution_state() == Thread::kThreadInNative);
  ASSERT(Dart_IsError(handle));
  Dart_PropagateError(handle);
  // We should never exit through normal control flow.
  UNREACHABLE();
}

// Not a leaf-function, throws error.
DEFINE_LEAF_RUNTIME_ENTRY(PropagateError,
                          /*argument_count=*/1,
                          DLRT_PropagateError);

DEFINE_RUNTIME_ENTRY(InitializeSharedField, 1) {
  // Running the initializer means running arbitrary Dart code that might yield
  // to a reload safepoint or have its active mutator slot stolen. Make sure we
  // likewise yield while waiting for the lock to avoid the holder of the lock
  // being blocked on locker-waiters for reload or a mutator slot.
  ReloadableStealableWriteRwLocker locker(
      thread, thread->isolate_group()->shared_field_initializer_rwlock());
  const Field& field = Field::CheckedHandle(zone, arguments.ArgAt(0));
  Object& result = Object::Handle(zone, field.StaticValue());
  if (result.ptr() == Object::sentinel().ptr()) {
    // Haven't lost a race to set the initial value.
    result = field.InitializeStatic();
    ThrowIfError(result);
    result = field.StaticValue();
    ASSERT(result.ptr() != Object::sentinel().ptr());
  }
  arguments.SetReturn(result);
}

// Throw if the value is not immutable.
//   Arg0: Value to check.
DEFINE_RUNTIME_ENTRY(EnsureDeeplyImmutable, 1) {
  const Instance& value = Instance::CheckedHandle(zone, arguments.ArgAt(0));
  value.EnsureDeeplyImmutable(zone);
}

#if defined(USING_MEMORY_SANITIZER)
extern "C" void dart_msan_read1(void* addr) {
  __msan_check_mem_is_initialized(addr, 1);
}
extern "C" void dart_msan_read2(void* addr) {
  __msan_check_mem_is_initialized(addr, 2);
}
extern "C" void dart_msan_read4(void* addr) {
  __msan_check_mem_is_initialized(addr, 4);
}
extern "C" void dart_msan_read8(void* addr) {
  __msan_check_mem_is_initialized(addr, 8);
}
extern "C" void dart_msan_read16(void* addr) {
  __msan_check_mem_is_initialized(addr, 16);
}
extern "C" void dart_msan_write1(void* addr) {
  __msan_unpoison(addr, 1);
}
extern "C" void dart_msan_write2(void* addr) {
  __msan_unpoison(addr, 2);
}
extern "C" void dart_msan_write4(void* addr) {
  __msan_unpoison(addr, 4);
}
extern "C" void dart_msan_write8(void* addr) {
  __msan_unpoison(addr, 8);
}
extern "C" void dart_msan_write16(void* addr) {
  __msan_unpoison(addr, 16);
}
#else
extern "C" void __msan_unpoison(const volatile void*, size_t) {
  UNREACHABLE();
}
extern "C" void __msan_unpoison_param(size_t) {
  UNREACHABLE();
}
#endif

#if !defined(USING_THREAD_SANITIZER)
extern "C" uint32_t __tsan_atomic32_load(uint32_t* addr, int order) {
  UNREACHABLE();
}
extern "C" void __tsan_atomic32_store(uint32_t* addr,
                                      uint32_t value,
                                      int order) {
  UNREACHABLE();
}
extern "C" uint64_t __tsan_atomic64_load(uint64_t* addr, int order) {
  UNREACHABLE();
}
extern "C" void __tsan_atomic64_store(uint64_t* addr,
                                      uint64_t value,
                                      int order) {
  UNREACHABLE();
}
extern "C" void __tsan_read1(void* addr) {
  UNREACHABLE();
}
extern "C" void __tsan_read2(void* addr) {
  UNREACHABLE();
}
extern "C" void __tsan_read4(void* addr) {
  UNREACHABLE();
}
extern "C" void __tsan_read8(void* addr) {
  UNREACHABLE();
}
extern "C" void __tsan_read16(void* addr) {
  UNREACHABLE();
}
extern "C" void __tsan_write1(void* addr) {
  UNREACHABLE();
}
extern "C" void __tsan_write2(void* addr) {
  UNREACHABLE();
}
extern "C" void __tsan_write4(void* addr) {
  UNREACHABLE();
}
extern "C" void __tsan_write8(void* addr) {
  UNREACHABLE();
}
extern "C" void __tsan_write16(void* addr) {
  UNREACHABLE();
}
extern "C" void __tsan_func_entry(void* pc) {
  UNREACHABLE();
}
extern "C" void __tsan_func_exit() {
  UNREACHABLE();
}
#else
#define CASE(x)                                                                \
  extern "C" NO_SANITIZE_THREAD DISABLE_SANITIZER_INSTRUMENTATION void         \
  dart_tsan_##x(void* addr) {                                                  \
    __tsan_##x##_pc(                                                           \
        addr, reinterpret_cast<void*>(                                         \
                  reinterpret_cast<uintptr_t>(__builtin_return_address(0)) |   \
                  kExternalPCBit));                                            \
  }

CASE(read1)
CASE(read2)
CASE(read4)
CASE(read8)
CASE(read16)
CASE(write1)
CASE(write2)
CASE(write4)
CASE(write8)
CASE(write16)
#undef CASE
extern "C" NO_SANITIZE_THREAD DISABLE_SANITIZER_INSTRUMENTATION void
dart_tsan_func_entry(void* pc) {
  __tsan_func_entry(reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(pc) |
                                            kExternalPCBit));
}
#endif

// These runtime entries are defined even when not using ASAN / MSAN / TSAN to
// keep offsets on Thread consistent.
DEFINE_LEAF_RUNTIME_ENTRY(MsanUnpoison, 2, __msan_unpoison);
DEFINE_LEAF_RUNTIME_ENTRY(MsanUnpoisonParam, 1, __msan_unpoison_param);
DEFINE_LEAF_RUNTIME_ENTRY(TsanAtomic32Load, 2, __tsan_atomic32_load);
DEFINE_LEAF_RUNTIME_ENTRY(TsanAtomic32Store, 3, __tsan_atomic32_store);
DEFINE_LEAF_RUNTIME_ENTRY(TsanAtomic64Load, 2, __tsan_atomic64_load);
DEFINE_LEAF_RUNTIME_ENTRY(TsanAtomic64Store, 3, __tsan_atomic64_store);
#if defined(USING_ADDRESS_SANITIZER)
DEFINE_LEAF_RUNTIME_ENTRY(SanRead1, 1, __asan_load1);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead2, 1, __asan_load2);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead4, 1, __asan_load4);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead8, 1, __asan_load8);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead16, 1, __asan_load16);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite1, 1, __asan_store1);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite2, 1, __asan_store2);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite4, 1, __asan_store4);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite8, 1, __asan_store8);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite16, 1, __asan_store16);
#elif defined(USING_MEMORY_SANITIZER)
DEFINE_LEAF_RUNTIME_ENTRY(SanRead1, 1, dart_msan_read1);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead2, 1, dart_msan_read2);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead4, 1, dart_msan_read4);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead8, 1, dart_msan_read8);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead16, 1, dart_msan_read16);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite1, 1, dart_msan_write1);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite2, 1, dart_msan_write2);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite4, 1, dart_msan_write4);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite8, 1, dart_msan_write8);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite16, 1, dart_msan_write16);
#elif defined(USING_THREAD_SANITIZER) && !defined(DART_PRECOMPILED_RUNTIME)
DEFINE_LEAF_RUNTIME_ENTRY(SanRead1, 1, dart_tsan_read1);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead2, 1, dart_tsan_read2);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead4, 1, dart_tsan_read4);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead8, 1, dart_tsan_read8);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead16, 1, dart_tsan_read16);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite1, 1, dart_tsan_write1);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite2, 1, dart_tsan_write2);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite4, 1, dart_tsan_write4);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite8, 1, dart_tsan_write8);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite16, 1, dart_tsan_write16);
#else
DEFINE_LEAF_RUNTIME_ENTRY(SanRead1, 1, __tsan_read1);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead2, 1, __tsan_read2);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead4, 1, __tsan_read4);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead8, 1, __tsan_read8);
DEFINE_LEAF_RUNTIME_ENTRY(SanRead16, 1, __tsan_read16);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite1, 1, __tsan_write1);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite2, 1, __tsan_write2);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite4, 1, __tsan_write4);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite8, 1, __tsan_write8);
DEFINE_LEAF_RUNTIME_ENTRY(SanWrite16, 1, __tsan_write16);
#endif
#if defined(USING_THREAD_SANITIZER) && !defined(DART_PRECOMPILED_RUNTIME)
DEFINE_LEAF_RUNTIME_ENTRY(TsanFuncEntry, 1, dart_tsan_func_entry);
#else
DEFINE_LEAF_RUNTIME_ENTRY(TsanFuncEntry, 1, __tsan_func_entry);
#endif
DEFINE_LEAF_RUNTIME_ENTRY(TsanFuncExit, 0, __tsan_func_exit);

}  // namespace dart