[cfe] Report cyclic typedef use in bounds of type parameters

Closes #43975.

Bug: https://github.com/dart-lang/sdk/issues/43975
Change-Id: Ia7e7b6c46d3c0d2eaa2e809008847d3f47897d2e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/182780
Reviewed-by: Johnni Winther <johnniwinther@google.com>
Commit-Queue: Dmitry Stefantsov <dmitryas@google.com>
diff --git a/pkg/front_end/lib/src/fasta/builder/function_type_builder.dart b/pkg/front_end/lib/src/fasta/builder/function_type_builder.dart
index 1f5bbe7..26f56fc 100644
--- a/pkg/front_end/lib/src/fasta/builder/function_type_builder.dart
+++ b/pkg/front_end/lib/src/fasta/builder/function_type_builder.dart
@@ -107,6 +107,8 @@
       typeParameters = <TypeParameter>[];
       for (TypeVariableBuilder t in typeVariables) {
         typeParameters.add(t.parameter);
+        // Build the bound to detect cycles in typedefs.
+        t.bound?.build(library, origin);
       }
     }
     return new FunctionType(positionalParameters, builtReturnType,
diff --git a/pkg/front_end/lib/src/fasta/builder/type_alias_builder.dart b/pkg/front_end/lib/src/fasta/builder/type_alias_builder.dart
index c4c5502..2f94d08 100644
--- a/pkg/front_end/lib/src/fasta/builder/type_alias_builder.dart
+++ b/pkg/front_end/lib/src/fasta/builder/type_alias_builder.dart
@@ -426,4 +426,18 @@
   }
 }
 
+/// Used to detect cycles in the declaration of a typedef
+///
+/// When a typedef is built, [pendingTypeAliasMarker] is used as a placeholder
+/// value to indicated that the process has started.  If somewhere in the
+/// process of building the typedef this value is encountered, it's replaced
+/// with [cyclicTypeAliasMarker] as the result of the build process.
+final InvalidType pendingTypeAliasMarker = new InvalidType();
+
+/// Used to detect cycles in the declaration of a typedef
+///
+/// When a typedef is built, [pendingTypeAliasMarker] is used as a placeholder
+/// value to indicated that the process has started.  If somewhere in the
+/// process of building the typedef this value is encountered, it's replaced
+/// with [cyclicTypeAliasMarker] as the result of the build process.
 final InvalidType cyclicTypeAliasMarker = new InvalidType();
diff --git a/pkg/front_end/lib/src/fasta/source/source_type_alias_builder.dart b/pkg/front_end/lib/src/fasta/source/source_type_alias_builder.dart
index 5f1a2d2..b84b276 100644
--- a/pkg/front_end/lib/src/fasta/source/source_type_alias_builder.dart
+++ b/pkg/front_end/lib/src/fasta/source/source_type_alias_builder.dart
@@ -128,17 +128,20 @@
 
   DartType buildThisType() {
     if (thisType != null) {
-      if (identical(thisType, cyclicTypeAliasMarker)) {
+      if (identical(thisType, pendingTypeAliasMarker)) {
+        thisType = cyclicTypeAliasMarker;
         library.addProblem(templateCyclicTypedef.withArguments(name),
             charOffset, noLength, fileUri);
         return const InvalidType();
+      } else if (identical(thisType, cyclicTypeAliasMarker)) {
+        return const InvalidType();
       }
       return thisType;
     }
     // It is a compile-time error for an alias (typedef) to refer to itself. We
     // detect cycles by detecting recursive calls to this method using an
     // instance of InvalidType that isn't identical to `const InvalidType()`.
-    thisType = cyclicTypeAliasMarker;
+    thisType = pendingTypeAliasMarker;
     TypeBuilder type = this.type;
     if (type != null) {
       DartType builtType =
@@ -150,7 +153,11 @@
             tv.bound?.build(library);
           }
         }
-        return thisType = builtType;
+        if (identical(thisType, cyclicTypeAliasMarker)) {
+          return thisType = const InvalidType();
+        } else {
+          return thisType = builtType;
+        }
       } else {
         return thisType = const InvalidType();
       }
diff --git a/pkg/front_end/testcases/general/issue43975.dart b/pkg/front_end/testcases/general/issue43975.dart
new file mode 100644
index 0000000..929affa
--- /dev/null
+++ b/pkg/front_end/testcases/general/issue43975.dart
@@ -0,0 +1,6 @@
+// Copyright (c) 2021, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+typedef F = void Function<X extends F>() Function();
+void main() {}
diff --git a/pkg/front_end/testcases/general/issue43975.dart.outline.expect b/pkg/front_end/testcases/general/issue43975.dart.outline.expect
new file mode 100644
index 0000000..a61aa23
--- /dev/null
+++ b/pkg/front_end/testcases/general/issue43975.dart.outline.expect
@@ -0,0 +1,13 @@
+library;
+//
+// Problems in library:
+//
+// pkg/front_end/testcases/general/issue43975.dart:5:9: Error: The typedef 'F' has a reference to itself.
+// typedef F = void Function<X extends F>() Function();
+//         ^
+//
+import self as self;
+
+typedef F = invalid-type;
+static method main() → void
+  ;
diff --git a/pkg/front_end/testcases/general/issue43975.dart.strong.expect b/pkg/front_end/testcases/general/issue43975.dart.strong.expect
new file mode 100644
index 0000000..a1da936
--- /dev/null
+++ b/pkg/front_end/testcases/general/issue43975.dart.strong.expect
@@ -0,0 +1,12 @@
+library;
+//
+// Problems in library:
+//
+// pkg/front_end/testcases/general/issue43975.dart:5:9: Error: The typedef 'F' has a reference to itself.
+// typedef F = void Function<X extends F>() Function();
+//         ^
+//
+import self as self;
+
+typedef F = invalid-type;
+static method main() → void {}
diff --git a/pkg/front_end/testcases/general/issue43975.dart.strong.transformed.expect b/pkg/front_end/testcases/general/issue43975.dart.strong.transformed.expect
new file mode 100644
index 0000000..a1da936
--- /dev/null
+++ b/pkg/front_end/testcases/general/issue43975.dart.strong.transformed.expect
@@ -0,0 +1,12 @@
+library;
+//
+// Problems in library:
+//
+// pkg/front_end/testcases/general/issue43975.dart:5:9: Error: The typedef 'F' has a reference to itself.
+// typedef F = void Function<X extends F>() Function();
+//         ^
+//
+import self as self;
+
+typedef F = invalid-type;
+static method main() → void {}
diff --git a/pkg/front_end/testcases/general/issue43975.dart.textual_outline.expect b/pkg/front_end/testcases/general/issue43975.dart.textual_outline.expect
new file mode 100644
index 0000000..ccb87d9
--- /dev/null
+++ b/pkg/front_end/testcases/general/issue43975.dart.textual_outline.expect
@@ -0,0 +1,2 @@
+typedef F = void Function<X extends F>() Function();
+void main() {}
diff --git a/pkg/front_end/testcases/general/issue43975.dart.textual_outline_modelled.expect b/pkg/front_end/testcases/general/issue43975.dart.textual_outline_modelled.expect
new file mode 100644
index 0000000..ccb87d9
--- /dev/null
+++ b/pkg/front_end/testcases/general/issue43975.dart.textual_outline_modelled.expect
@@ -0,0 +1,2 @@
+typedef F = void Function<X extends F>() Function();
+void main() {}
diff --git a/pkg/front_end/testcases/general/three_typedefs_loop.dart.outline.expect b/pkg/front_end/testcases/general/three_typedefs_loop.dart.outline.expect
index 12b78a9..f030616 100644
--- a/pkg/front_end/testcases/general/three_typedefs_loop.dart.outline.expect
+++ b/pkg/front_end/testcases/general/three_typedefs_loop.dart.outline.expect
@@ -13,7 +13,7 @@
 import self as self;
 import "dart:core" as core;
 
-typedef Foo<unrelated T extends core::Object* = dynamic> = (((invalid-type) →* void) →* void) →* void;
+typedef Foo<unrelated T extends core::Object* = dynamic> = invalid-type;
 typedef Bar<unrelated T extends core::Object* = dynamic> = ((invalid-type) →* void) →* void;
 typedef Baz<unrelated T extends core::Object* = dynamic> = (invalid-type) →* void;
 static method main() → dynamic
diff --git a/pkg/front_end/testcases/general/three_typedefs_loop.dart.strong.expect b/pkg/front_end/testcases/general/three_typedefs_loop.dart.strong.expect
index 3773b6f..bd2fff9 100644
--- a/pkg/front_end/testcases/general/three_typedefs_loop.dart.strong.expect
+++ b/pkg/front_end/testcases/general/three_typedefs_loop.dart.strong.expect
@@ -13,7 +13,7 @@
 import self as self;
 import "dart:core" as core;
 
-typedef Foo<unrelated T extends core::Object* = dynamic> = (((invalid-type) →* void) →* void) →* void;
+typedef Foo<unrelated T extends core::Object* = dynamic> = invalid-type;
 typedef Bar<unrelated T extends core::Object* = dynamic> = ((invalid-type) →* void) →* void;
 typedef Baz<unrelated T extends core::Object* = dynamic> = (invalid-type) →* void;
 static method main() → dynamic {}
diff --git a/pkg/front_end/testcases/general/three_typedefs_loop.dart.strong.transformed.expect b/pkg/front_end/testcases/general/three_typedefs_loop.dart.strong.transformed.expect
index 3773b6f..bd2fff9 100644
--- a/pkg/front_end/testcases/general/three_typedefs_loop.dart.strong.transformed.expect
+++ b/pkg/front_end/testcases/general/three_typedefs_loop.dart.strong.transformed.expect
@@ -13,7 +13,7 @@
 import self as self;
 import "dart:core" as core;
 
-typedef Foo<unrelated T extends core::Object* = dynamic> = (((invalid-type) →* void) →* void) →* void;
+typedef Foo<unrelated T extends core::Object* = dynamic> = invalid-type;
 typedef Bar<unrelated T extends core::Object* = dynamic> = ((invalid-type) →* void) →* void;
 typedef Baz<unrelated T extends core::Object* = dynamic> = (invalid-type) →* void;
 static method main() → dynamic {}
diff --git a/pkg/front_end/testcases/general_nnbd_opt_out/three_typedefs_loop.dart.weak.expect b/pkg/front_end/testcases/general_nnbd_opt_out/three_typedefs_loop.dart.weak.expect
index de06f6f..d648775a 100644
--- a/pkg/front_end/testcases/general_nnbd_opt_out/three_typedefs_loop.dart.weak.expect
+++ b/pkg/front_end/testcases/general_nnbd_opt_out/three_typedefs_loop.dart.weak.expect
@@ -13,7 +13,7 @@
 import self as self;
 import "dart:core" as core;
 
-typedef Foo<unrelated T extends core::Object* = dynamic> = (((invalid-type) →* void) →* void) →* void;
+typedef Foo<unrelated T extends core::Object* = dynamic> = invalid-type;
 typedef Bar<unrelated T extends core::Object* = dynamic> = ((invalid-type) →* void) →* void;
 typedef Baz<unrelated T extends core::Object* = dynamic> = (invalid-type) →* void;
 static method main() → dynamic {}
diff --git a/pkg/front_end/testcases/general_nnbd_opt_out/three_typedefs_loop.dart.weak.transformed.expect b/pkg/front_end/testcases/general_nnbd_opt_out/three_typedefs_loop.dart.weak.transformed.expect
index de06f6f..d648775a 100644
--- a/pkg/front_end/testcases/general_nnbd_opt_out/three_typedefs_loop.dart.weak.transformed.expect
+++ b/pkg/front_end/testcases/general_nnbd_opt_out/three_typedefs_loop.dart.weak.transformed.expect
@@ -13,7 +13,7 @@
 import self as self;
 import "dart:core" as core;
 
-typedef Foo<unrelated T extends core::Object* = dynamic> = (((invalid-type) →* void) →* void) →* void;
+typedef Foo<unrelated T extends core::Object* = dynamic> = invalid-type;
 typedef Bar<unrelated T extends core::Object* = dynamic> = ((invalid-type) →* void) →* void;
 typedef Baz<unrelated T extends core::Object* = dynamic> = (invalid-type) →* void;
 static method main() → dynamic {}
diff --git a/pkg/front_end/testcases/rasta/malformed_function_type.dart.outline.expect b/pkg/front_end/testcases/rasta/malformed_function_type.dart.outline.expect
index 6c35ab9..dc3f69b 100644
--- a/pkg/front_end/testcases/rasta/malformed_function_type.dart.outline.expect
+++ b/pkg/front_end/testcases/rasta/malformed_function_type.dart.outline.expect
@@ -7,8 +7,7 @@
 //                ^
 //
 import self as self;
-import "dart:core" as core;
 
-typedef Handle = (core::String*) →* invalid-type;
+typedef Handle = invalid-type;
 static method main() → dynamic
   ;
diff --git a/pkg/front_end/testcases/rasta/malformed_function_type.dart.strong.expect b/pkg/front_end/testcases/rasta/malformed_function_type.dart.strong.expect
index 1cb9f39..3444c42 100644
--- a/pkg/front_end/testcases/rasta/malformed_function_type.dart.strong.expect
+++ b/pkg/front_end/testcases/rasta/malformed_function_type.dart.strong.expect
@@ -7,9 +7,8 @@
 //                ^
 //
 import self as self;
-import "dart:core" as core;
 
-typedef Handle = (core::String*) →* invalid-type;
+typedef Handle = invalid-type;
 static method main() → dynamic {
-  (core::String*) →* invalid-type h;
+  invalid-type h;
 }
diff --git a/pkg/front_end/testcases/rasta/malformed_function_type.dart.strong.transformed.expect b/pkg/front_end/testcases/rasta/malformed_function_type.dart.strong.transformed.expect
index 1cb9f39..3444c42 100644
--- a/pkg/front_end/testcases/rasta/malformed_function_type.dart.strong.transformed.expect
+++ b/pkg/front_end/testcases/rasta/malformed_function_type.dart.strong.transformed.expect
@@ -7,9 +7,8 @@
 //                ^
 //
 import self as self;
-import "dart:core" as core;
 
-typedef Handle = (core::String*) →* invalid-type;
+typedef Handle = invalid-type;
 static method main() → dynamic {
-  (core::String*) →* invalid-type h;
+  invalid-type h;
 }