[vm] Fix TSAN crashes due to TSAN's own overflow.
TEST=vm/cc/DartAPI_StackOverflow
Change-Id: If5456f4f0a194332b38901f9bd677f3cd9af561c
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/456421
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Alexander Markov <alexmarkov@google.com>
diff --git a/runtime/vm/os_thread.cc b/runtime/vm/os_thread.cc
index e38d914..07cff87 100644
--- a/runtime/vm/os_thread.cc
+++ b/runtime/vm/os_thread.cc
@@ -7,6 +7,7 @@
#include "platform/address_sanitizer.h"
#include "platform/atomic.h"
#include "platform/memory_sanitizer.h"
+#include "platform/thread_sanitizer.h"
#include "vm/lockers.h"
#include "vm/log.h"
#include "vm/thread_interrupter.h"
@@ -52,6 +53,20 @@
stack_headroom_ = CalculateHeadroom(stack_base_ - stack_limit_);
+#if defined(USING_THREAD_SANITIZER)
+ // The Mac default stack size of 8 MB results in TSAN's stack recording
+ // overflowing in some tests before Dart reaches regular stack overflow.
+ // Clamp the stack size used by Dart so we hit a Dart stack overflow
+ // exception before corrupting TSAN. This is only relevant in run_vm_tests,
+ // which runs Dart on the main thread. The standalone embedder runs even the
+ // main isolate on thread pool workers, which the VM created with a smaller
+ // stack size.
+ const size_t kMaxDartStackSize = GetMaxStackSize();
+ if (stack_base_ - stack_limit_ - stack_headroom_ > kMaxDartStackSize) {
+ stack_headroom_ = stack_base_ - stack_limit_ - kMaxDartStackSize;
+ }
+#endif
+
ASSERT(stack_base_ != 0);
ASSERT(stack_limit_ != 0);
ASSERT(stack_base_ > stack_limit_);