Relands: Refactors RBE support (#49660)
Relands https://github.com/flutter/engine/pull/49416
diff --git a/.gitignore b/.gitignore
index fcd681e..59a4590 100644
--- a/.gitignore
+++ b/.gitignore
@@ -130,4 +130,9 @@
# Prebuilt binaries.
/prebuilts/
+
+# GN build support for protobufs vended by Fuchsia.
/build/secondary/third_party/protobuf
+
+# RBE support configurations and scripts vended from CIPD
+/build/rbe
diff --git a/DEPS b/DEPS
index d7f7fe9..3e9d636 100644
--- a/DEPS
+++ b/DEPS
@@ -53,7 +53,7 @@
# https://chrome-infra-packages.appspot.com/p/fuchsia/third_party/goma/client
'goma_version': ' git_revision:41b3bcb64014144a844153fd5588c36411fffb56',
- 'reclient_version': 'git_revision:81e819b39d4743462857cc55430d898b9fcca1af',
+ 'reclient_version': 'git_revision:f3883c2237b0eb9cc9524cb571b5ab8378f257e4',
'gcloud_version': 'version:2@444.0.0.chromium.3',
@@ -126,6 +126,10 @@
# specified by GOMA_DIR, or installed in the default goma install location.
'use_cipd_goma': False,
+ # When this is true, the Flutter Engine's configuration files and scripts for
+ # RBE will be downloaded from CIPD. This option is only usable by Googlers.
+ 'use_rbe': False,
+
# This is not downloaded be default because it increases the
# `gclient sync` time by between 1 and 3 minutes. This option is enabled
# in flutter/ci/builders/mac_impeller_cmake_example.json, and is likely to
@@ -266,7 +270,7 @@
]
deps = {
- 'src': 'https://github.com/flutter/buildroot.git' + '@' + '8c274b21f1ad4f2aec0a5e0ae8f4264393045b4b',
+ 'src': 'https://github.com/flutter/buildroot.git' + '@' + 'a43582b52d361bc3da156a8e6eab6dd947ca339d',
'src/flutter/third_party/rapidjson':
Var('flutter_git') + '/third_party/rapidjson' + '@' + 'ef3564c5c8824989393b87df25355baf35ff544b',
@@ -843,7 +847,7 @@
'version': Var('clang_version'),
}
],
- 'condition': 'host_os == "linux" and host_cpu == "x64"',
+ 'condition': 'host_os == "linux" or host_os == "mac"',
'dep_type': 'cipd',
},
@@ -903,7 +907,7 @@
'dep_type': 'cipd',
},
- # reclient.
+ # RBE binaries and configs.
'src/buildtools/linux-x64/reclient': {
'packages': [
{
@@ -911,7 +915,40 @@
'version': Var('reclient_version'),
}
],
- 'condition': 'host_os == "linux" and host_cpu == "x64"',
+ 'condition': 'use_rbe and host_os == "linux" and host_cpu == "x64"',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/buildtools/mac-arm64/reclient': {
+ 'packages': [
+ {
+ 'package': 'infra/rbe/client/${{platform}}',
+ 'version': Var('reclient_version'),
+ }
+ ],
+ 'condition': 'use_rbe and host_os == "mac" and host_cpu == "arm64"',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/buildtools/mac-x64/reclient': {
+ 'packages': [
+ {
+ 'package': 'infra/rbe/client/${{platform}}',
+ 'version': Var('reclient_version'),
+ }
+ ],
+ 'condition': 'use_rbe and host_os == "mac" and host_cpu == "x64"',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/flutter/build/rbe': {
+ 'packages': [
+ {
+ 'package': 'flutter_internal/rbe/reclient_cfgs',
+ 'version': 'U42C0v8jI-_YREjd8rbDEt0evvqvLWJ_NTkaiJ_Clt8C',
+ }
+ ],
+ 'condition': 'use_rbe',
'dep_type': 'cipd',
},
@@ -923,7 +960,18 @@
'version': Var('gcloud_version'),
}
],
- 'condition': 'host_os == "linux" and host_cpu == "x64"',
+ 'condition': 'use_rbe and host_os == "linux" and host_cpu == "x64"',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/buildtools/mac-arm64/gcloud': {
+ 'packages': [
+ {
+ 'package': 'infra/3pp/tools/gcloud/${{platform}}',
+ 'version': Var('gcloud_version'),
+ }
+ ],
+ 'condition': 'use_rbe and host_os == "mac" and host_cpu == "arm64"',
'dep_type': 'cipd',
},
diff --git a/ci/builders/linux_android_emulator.json b/ci/builders/linux_android_emulator.json
index 847d161ed..bc5217a 100644
--- a/ci/builders/linux_android_emulator.json
+++ b/ci/builders/linux_android_emulator.json
@@ -7,6 +7,9 @@
"kvm=1",
"cores=8"
],
+ "gclient_variables": {
+ "use_rbe": true
+ },
"gn": [
"--android",
"--android-cpu=x64",
@@ -69,6 +72,9 @@
"kvm=1",
"cores=8"
],
+ "gclient_variables": {
+ "use_rbe": true
+ },
"gn": [
"--android",
"--android-cpu=x86",
diff --git a/ci/builders/linux_unopt.json b/ci/builders/linux_unopt.json
index 32b9e02..89bd717 100644
--- a/ci/builders/linux_unopt.json
+++ b/ci/builders/linux_unopt.json
@@ -6,6 +6,9 @@
"os=Linux",
"cores=32"
],
+ "gclient_variables": {
+ "use_rbe": true
+ },
"gn": [
"--runtime-mode",
"debug",
@@ -84,6 +87,9 @@
"os=Linux"
],
"dependencies": [],
+ "gclient_variables": {
+ "use_rbe": true
+ },
"gn": [
"--android",
"--embedder-for-target",
@@ -112,6 +118,9 @@
"version": "last_updated:2023-02-03T15:32:01-0800"
}
],
+ "gclient_variables": {
+ "use_rbe": true
+ },
"gn": [
"--android",
"--unoptimized",
diff --git a/ci/builders/mac_unopt.json b/ci/builders/mac_unopt.json
index 078b165..359911e 100644
--- a/ci/builders/mac_unopt.json
+++ b/ci/builders/mac_unopt.json
@@ -16,7 +16,8 @@
"mac_model=Macmini8,1"
],
"gclient_variables": {
- "download_android_deps": false
+ "download_android_deps": false,
+ "use_rbe": true
},
"gn": [
"--runtime-mode",
@@ -24,7 +25,10 @@
"--unoptimized",
"--no-lto",
"--prebuilt-dart-sdk",
- "--enable-impeller-3d"
+ "--enable-impeller-3d",
+ "--rbe",
+ "--no-goma",
+ "--xcode-symlinks"
],
"name": "host_debug_unopt",
"ninja": {
@@ -75,14 +79,18 @@
"cpu=x86"
],
"gclient_variables": {
- "download_android_deps": false
+ "download_android_deps": false,
+ "use_rbe": true
},
"gn": [
"--ios",
"--runtime-mode",
"debug",
"--simulator",
- "--no-lto"
+ "--no-lto",
+ "--rbe",
+ "--no-goma",
+ "--xcode-symlinks"
],
"name": "ios_debug_sim",
"ninja": {
@@ -132,7 +140,8 @@
"cpu=arm64"
],
"gclient_variables": {
- "download_android_deps": false
+ "download_android_deps": false,
+ "use_rbe": true
},
"gn": [
"--runtime-mode",
@@ -142,7 +151,10 @@
"--prebuilt-dart-sdk",
"--force-mac-arm64",
"--mac-cpu",
- "arm64"
+ "arm64",
+ "--rbe",
+ "--no-goma",
+ "--xcode-symlinks"
],
"name": "host_debug_unopt_arm64",
"ninja": {
@@ -181,7 +193,8 @@
"cpu=arm64"
],
"gclient_variables": {
- "download_android_deps": false
+ "download_android_deps": false,
+ "use_rbe": true
},
"gn": [
"--ios",
@@ -191,7 +204,10 @@
"--no-lto",
"--force-mac-arm64",
"--simulator-cpu",
- "arm64"
+ "arm64",
+ "--rbe",
+ "--no-goma",
+ "--xcode-symlinks"
],
"name": "ios_debug_sim_arm64",
"ninja": {
@@ -251,7 +267,8 @@
"cpu=arm64"
],
"gclient_variables": {
- "download_android_deps": false
+ "download_android_deps": false,
+ "use_rbe": true
},
"gn": [
"--ios",
@@ -262,7 +279,10 @@
"--force-mac-arm64",
"--simulator-cpu",
"arm64",
- "--darwin-extension-safe"
+ "--darwin-extension-safe",
+ "--rbe",
+ "--no-goma",
+ "--xcode-symlinks"
],
"name": "ios_debug_sim_arm64_extension_safe",
"ninja": {
diff --git a/ci/builders/standalone/linux_benchmarks.json b/ci/builders/standalone/linux_benchmarks.json
index a408d9c..b118daf 100644
--- a/ci/builders/standalone/linux_benchmarks.json
+++ b/ci/builders/standalone/linux_benchmarks.json
@@ -4,7 +4,8 @@
"os=Linux"
],
"gclient_variables": {
- "download_android_deps": false
+ "download_android_deps": false,
+ "use_rbe": true
},
"gn": [
"--runtime-mode",
diff --git a/tools/gn b/tools/gn
index 6ceda84..7b9b2fe 100755
--- a/tools/gn
+++ b/tools/gn
@@ -205,8 +205,6 @@
host_cpu = get_host_cpu()
if host_os == 'win':
host_os = 'windows'
- if host_os == 'mac' and host_cpu == 'arm64':
- host_cpu = 'x64'
return '%s-%s' % (host_os, host_cpu)
@@ -217,9 +215,10 @@
if not args.rbe:
return rbe_gn_args
- if get_host_os() not in ['linux']:
+ if get_host_os() not in ['linux', 'mac']:
print(
- 'The --rbe flag has no effect. RBE is currently only supported on Linux.'
+ 'The --rbe flag has no effect. RBE is currently only supported on '
+ 'macOS and Linux.'
)
return rbe_gn_args
@@ -239,7 +238,9 @@
)
bootstrap_path = os.path.join(cipd_reclient_dir, 'bootstrap')
reproxy_path = os.path.join(cipd_reclient_dir, 'reproxy')
- rbe_cfg_path = os.path.join(SRC_ROOT, 'build', 'rbe.cfg')
+ rbe_cfg_path = os.path.join(
+ SRC_ROOT, 'flutter', 'build', 'rbe', 'reclient.cfg'
+ )
bootstrap_cmd = [
bootstrap_path,
'--re_proxy=' + reproxy_path,
@@ -260,8 +261,20 @@
rbe_gn_args['rbe_dial_timeout'] = args.rbe_dial_timeout
if args.rbe_platform:
rbe_gn_args['rbe_platform'] = args.rbe_platform
- if args.rbe_dir:
- rbe_gn_args['rbe_dir'] = args.rbe_dir
+
+ rbe_gn_args['rbe_dir'] = os.path.join(
+ SRC_ROOT, 'buildtools', buildtools_dir(), 'reclient'
+ )
+
+ rbe_gn_args['rbe_cfg'] = os.path.join(
+ SRC_ROOT, 'flutter', 'build', 'rbe',
+ 'rewrapper-' + buildtools_dir() + '.cfg'
+ )
+
+ if sys.platform == 'darwin':
+ if (not running_on_luci or args.xcode_symlinks or
+ os.getenv('FLUTTER_GOMA_CREATE_XCODE_SYMLINKS', '0') == '1'):
+ rbe_gn_args['create_xcode_symlinks'] = True
return rbe_gn_args
@@ -285,8 +298,16 @@
# care of starting and stopping the compiler proxy.
running_on_luci = os.environ.get('LUCI_CONTEXT') is not None
+ # The GOMA client has no arm64 binary, so run the x64 binary through
+ # Rosetta.
+ buildtools_platform = buildtools_dir()
+ if buildtools_platform == 'mac-arm64':
+ buildtools_platform = 'mac-x64'
+
# Prefer the goma fetched by gclient if it exists.
- cipd_goma_dir = os.path.join(SRC_ROOT, 'buildtools', buildtools_dir(), 'goma')
+ cipd_goma_dir = os.path.join(
+ SRC_ROOT, 'buildtools', buildtools_platform, 'goma'
+ )
# Next, if GOMA_DIR is set, use that install.
goma_dir = os.environ.get('GOMA_DIR')
