[VM/Runtime] Check return value from message handler 'run' method.

TEST=cq

Change-Id: I770e1c606c6622e5cb63bca72b74c63ce776ef53
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/210428
Reviewed-by: Ryan Macnak <rmacnak@google.com>
Commit-Queue: Siva Annamalai <asiva@google.com>
diff --git a/runtime/vm/dart_api_impl.cc b/runtime/vm/dart_api_impl.cc
index 00061db..5d21abf 100644
--- a/runtime/vm/dart_api_impl.cc
+++ b/runtime/vm/dart_api_impl.cc
@@ -1972,6 +1972,7 @@
 
 DART_EXPORT Dart_Handle Dart_RunLoop() {
   Isolate* I;
+  bool result;
   {
     Thread* T = Thread::Current();
     I = T->isolate();
@@ -1988,14 +1989,21 @@
     RunLoopData data;
     data.monitor = &monitor;
     data.done = false;
-    I->message_handler()->Run(I->group()->thread_pool(), NULL, RunLoopDone,
-                              reinterpret_cast<uword>(&data));
-    while (!data.done) {
-      ml.Wait();
+    result =
+        I->message_handler()->Run(I->group()->thread_pool(), NULL, RunLoopDone,
+                                  reinterpret_cast<uword>(&data));
+    if (result) {
+      while (!data.done) {
+        ml.Wait();
+      }
     }
   }
   ::Dart_EnterIsolate(Api::CastIsolate(I));
-  if (I->sticky_error() != Object::null()) {
+  if (!result) {
+    Thread* T = Thread::Current();
+    TransitionNativeToVM transition(T);
+    return Api::NewError("Run method in isolate message handler failed");
+  } else if (I->sticky_error() != Object::null()) {
     Thread* T = Thread::Current();
     TransitionNativeToVM transition(T);
     return Api::NewHandle(T, I->StealStickyError());
diff --git a/runtime/vm/message_handler.cc b/runtime/vm/message_handler.cc
index 28a89a2..bbb8211 100644
--- a/runtime/vm/message_handler.cc
+++ b/runtime/vm/message_handler.cc
@@ -100,7 +100,7 @@
   // By default, there is no custom message notification.
 }
 
-void MessageHandler::Run(ThreadPool* pool,
+bool MessageHandler::Run(ThreadPool* pool,
                          StartCallback start_callback,
                          EndCallback end_callback,
                          CallbackData data) {
@@ -118,8 +118,15 @@
   end_callback_ = end_callback;
   callback_data_ = data;
   task_running_ = true;
-  const bool launched_successfully = pool_->Run<MessageHandlerTask>(this);
-  ASSERT(launched_successfully);
+  bool result = pool_->Run<MessageHandlerTask>(this);
+  if (!result) {
+    pool_ = nullptr;
+    start_callback_ = nullptr;
+    end_callback_ = nullptr;
+    callback_data_ = 0;
+    task_running_ = false;
+  }
+  return result;
 }
 
 void MessageHandler::PostMessage(std::unique_ptr<Message> message,
diff --git a/runtime/vm/message_handler.h b/runtime/vm/message_handler.h
index 19b5101..715641b 100644
--- a/runtime/vm/message_handler.h
+++ b/runtime/vm/message_handler.h
@@ -48,7 +48,10 @@
   // no longer has any live ports.  Abnormal termination occurs when
   // HandleMessage() indicates that an error has occurred during
   // message processing.
-  void Run(ThreadPool* pool,
+
+  // Returns false if the handler terminated abnormally, otherwise it
+  // returns true.
+  bool Run(ThreadPool* pool,
            StartCallback start_callback,
            EndCallback end_callback,
            CallbackData data);
diff --git a/runtime/vm/native_api_impl.cc b/runtime/vm/native_api_impl.cc
index 07a2fec..793accb 100644
--- a/runtime/vm/native_api_impl.cc
+++ b/runtime/vm/native_api_impl.cc
@@ -90,8 +90,13 @@
 
   NativeMessageHandler* nmh = new NativeMessageHandler(name, handler);
   Dart_Port port_id = PortMap::CreatePort(nmh);
-  PortMap::SetPortState(port_id, PortMap::kLivePort);
-  nmh->Run(Dart::thread_pool(), NULL, NULL, 0);
+  if (port_id != ILLEGAL_PORT) {
+    PortMap::SetPortState(port_id, PortMap::kLivePort);
+    if (!nmh->Run(Dart::thread_pool(), NULL, NULL, 0)) {
+      PortMap::ClosePort(port_id);
+      port_id = ILLEGAL_PORT;
+    }
+  }
   Dart::ResetActiveApiCall();
   return port_id;
 }