Changes to support RBE in Windows builds (#820)
This PR does a few things:
1. It changes how the Windows toolchain and SDK are fetched and where
they are stored. Previously, the Windows toolchain and SDK were
downloaded under `depot_tools`, which typically exists outside of the
engine source root both locally and in CI. That setup makes the Windows
SDK inaccessible to RBE builds. Instead, this PR checks out
`depot_tools` under `flutter/third_party/depot_tools` by listing it in
the `DEPS` file. This location is under the engine source root, and is
therefore accessible to RBE builds. This change has the downside of
breaking how CI does caching of the `depot_tools` Windows toolchain and
SDK, which will add some time to builds.
2. It updates Windows GN toolchain setup logic to match what is in the
Chrome tree. Updates to `build/find_depot_tools.py`,
`build/toolchain/win/BUILD.gn`, and
`build/toolchain/win/setup_toolchain.py` are copied from there and
adapted to the Flutter engine repo.
diff --git a/build/.gitignore b/build/.gitignore
index 48b9b2e..05efc47 100644
--- a/build/.gitignore
+++ b/build/.gitignore
@@ -1,3 +1,4 @@
# Generated file containing information about the VS toolchain on Windows
win_toolchain.json
+new_win_toolchain.json
/linux/debian_*-sysroot/
diff --git a/build/config/BUILDCONFIG.gn b/build/config/BUILDCONFIG.gn
index fabc259..87ea4c5 100644
--- a/build/config/BUILDCONFIG.gn
+++ b/build/config/BUILDCONFIG.gn
@@ -526,13 +526,9 @@
host_toolchain = "//build/toolchain/linux:clang_$host_cpu"
set_default_toolchain("//build/toolchain/custom")
} else if (is_win) {
- if (is_clang) {
- host_toolchain = "//build/toolchain/win:clang_$host_cpu"
- set_default_toolchain("//build/toolchain/win:clang_$current_cpu")
- } else {
- host_toolchain = "//build/toolchain/win:$host_cpu"
- set_default_toolchain("//build/toolchain/win:$current_cpu")
- }
+ assert(is_clang)
+ host_toolchain = "//build/toolchain/win:clang_$host_cpu"
+ set_default_toolchain("//build/toolchain/win:clang_$current_cpu")
} else if (is_android) {
if (host_os == "linux") {
# Use clang for the x86/64 Linux host builds.
@@ -544,11 +540,8 @@
} else if (host_os == "mac") {
host_toolchain = "//build/toolchain/mac:clang_$host_cpu"
} else if (host_os == "win") {
- if (is_clang) {
- host_toolchain = "//build/toolchain/win:clang_$host_cpu"
- } else {
- host_toolchain = "//build/toolchain/win:$host_cpu"
- }
+ assert(is_clang)
+ host_toolchain = "//build/toolchain/win:clang_$host_cpu"
} else {
assert(false, "Unknown host for android cross compile")
}
diff --git a/build/find_depot_tools.py b/build/find_depot_tools.py
index 5d6661d..11e2f9a 100644
--- a/build/find_depot_tools.py
+++ b/build/find_depot_tools.py
@@ -1,23 +1,41 @@
-# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+#!/usr/bin/env python3
+# Copyright 2011 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Small utility function to find depot_tools and add it to the python path.
Will throw an ImportError exception if depot_tools can't be found since it
imports breakpad.
+
+This can also be used as a standalone script to print out the depot_tools
+directory location.
"""
+
import os
import sys
+# Path to //src
+SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+
+
def IsRealDepotTools(path):
- return os.path.isfile(os.path.join(path, 'gclient.py'))
+ expanded_path = os.path.expanduser(path)
+ return os.path.isfile(os.path.join(expanded_path, 'gclient.py'))
def add_depot_tools_to_path():
"""Search for depot_tools and add it to sys.path."""
- # First look if depot_tools is already in PYTHONPATH.
+ # First, check if we have a DEPS'd in "depot_tools".
+ deps_depot_tools = os.path.join(SRC, 'flutter', 'third_party', 'depot_tools')
+ if IsRealDepotTools(deps_depot_tools):
+ # Put the pinned version at the start of the sys.path, in case there
+ # are other non-pinned versions already on the sys.path.
+ sys.path.insert(0, deps_depot_tools)
+ return deps_depot_tools
+
+ # Then look if depot_tools is already in PYTHONPATH.
for i in sys.path:
if i.rstrip(os.sep).endswith('depot_tools') and IsRealDepotTools(i):
return i
@@ -39,7 +57,18 @@
print('Failed to find depot_tools', file=sys.stderr)
return None
-add_depot_tools_to_path()
+DEPOT_TOOLS_PATH = add_depot_tools_to_path()
# pylint: disable=W0611
import breakpad
+
+
+def main():
+ if DEPOT_TOOLS_PATH is None:
+ return 1
+ print(DEPOT_TOOLS_PATH)
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/build/toolchain/win/BUILD.gn b/build/toolchain/win/BUILD.gn
index 79269b7..46d28fb 100644
--- a/build/toolchain/win/BUILD.gn
+++ b/build/toolchain/win/BUILD.gn
@@ -2,6 +2,9 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+import("//build/toolchain/rbe.gni")
+import("//build/toolchain/win/win_toolchain_data.gni")
+
default_clang_base_path = "//buildtools/windows-x64/clang"
declare_args() {
@@ -31,23 +34,18 @@
# This tool will is used as a wrapper for various commands below.
tool_wrapper_path = rebase_path("tool_wrapper.py", root_build_dir)
-toolchain_data = exec_script("setup_toolchain.py",
- [
- visual_studio_path,
- windows_sdk_path,
- visual_studio_runtime_dirs,
- current_cpu,
- ],
- "scope")
-
-if (vc_bin_dir == "") {
- vc_bin_dir = toolchain_data.vc_bin_dir
-}
-
if (use_goma) {
- goma_prefix = "$goma_dir/gomacc.exe "
+ compiler_prefix = "$goma_dir/gomacc.exe "
+ asm_prefix = "$goma_dir/gomacc.exe "
+} else if (use_rbe) {
+ compiler_args = rewrapper_command + [
+ "--labels=type=compile,compiler=clang-cl,lang=cpp ",
+ ]
+ compiler_prefix = string_join(" ", compiler_args)
+ asm_prefix = " "
} else {
- goma_prefix = ""
+ compiler_prefix = ""
+ asm_prefix = ""
}
if (current_toolchain == default_toolchain) {
@@ -80,7 +78,18 @@
}
}
+ # TODO(zanderso): Assert that clang is always used or remove this logic.
+ if (defined(toolchain_args.is_clang)) {
+ toolchain_is_clang = toolchain_args.is_clang
+ } else {
+ toolchain_is_clang = is_clang
+ }
+
env = invoker.environment
+ env_path = "$root_out_dir/$env"
+ print(env_path)
+ env_file = read_file(env_path, "string")
+ print(env_file)
cl = invoker.cl
@@ -88,40 +97,54 @@
lib_switch = ""
lib_dir_switch = "/LIBPATH:"
- tool("cc") {
- rspfile = "{{output}}.rsp"
+ # If possible, pass system includes as flags to the compiler. When that's
+ # not possible, load a full environment file (containing %INCLUDE% and
+ # %PATH%) -- e.g. 32-bit MSVS builds require %PATH% to be set and just
+ # passing in a list of include directories isn't enough.
+ if (defined(invoker.sys_include_flags)) {
+ assert(toolchain_is_clang)
+ env_wrapper = ""
+ sys_include_flags =
+ "${invoker.sys_include_flags} " # Note trailing space.
+ } else {
+ # clang-cl doesn't need this env hoop, so omit it there.
+ assert(!toolchain_is_clang)
+ env_wrapper = "ninja -t msvc -e $env -- " # Note trailing space.
+ sys_include_flags = ""
+ }
- # TODO(brettw) enable this when GN support in the binary has been rolled.
- #precompiled_header_type = "msvc"
- pdbname = "{{target_out_dir}}/{{target_output_name}}_c.pdb"
- command = "ninja -t msvc -e $env -- $cl /nologo /showIncludes @$rspfile /c {{source}} /Fo{{output}} /Fd$pdbname"
+ tool("cc") {
+ precompiled_header_type = "msvc"
+ pdbname = "{{target_out_dir}}/{{label_name}}_c.pdb"
depsformat = "msvc"
description = "CC {{output}}"
- outputs = [
- "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.obj",
- ]
- rspfile_content = "{{defines}} {{include_dirs}} {{cflags}} {{cflags_c}}"
+ outputs = [ "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.obj" ]
+
+ # Label names may have spaces in them so the pdbname must be quoted. The
+ # source and output don't need to be quoted because GN knows they're a
+ # full file name and will quote automatically when necessary.
+ command = "$env_wrapper$cl /c {{source}} /nologo /showIncludes:user $sys_include_flags{{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} /Fo{{output}} /Fd\"$pdbname\""
}
tool("cxx") {
- rspfile = "{{output}}.rsp"
-
- # TODO(brettw) enable this when GN support in the binary has been rolled.
- #precompiled_header_type = "msvc"
+ precompiled_header_type = "msvc"
# The PDB name needs to be different between C and C++ compiled files.
- pdbname = "{{target_out_dir}}/{{target_output_name}}_cc.pdb"
+ pdbname = "{{target_out_dir}}/{{label_name}}_cc.pdb"
+
+ # TODO(zanderso): This logic should be moved to be with the other compiler
+ # flag logic in build/config/compiler/BUILD.gn or so.
flags = ""
if (is_clang && invoker.current_cpu == "x86") {
flags = "-m32"
}
- command = "ninja -t msvc -e $env -- $cl $flags /nologo /showIncludes @$rspfile /c {{source}} /Fo{{output}} /Fd$pdbname"
+
depsformat = "msvc"
description = "CXX {{output}}"
- outputs = [
- "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.obj",
- ]
- rspfile_content = "{{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}}"
+ outputs = [ "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.obj" ]
+
+ # See comment in CC tool about quoting.
+ command = "$env_wrapper$cl /c {{source}} /Fo{{output}} $flags /nologo /showIncludes:user $sys_include_flags{{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} /Fd\"$pdbname\""
}
tool("rc") {
@@ -141,7 +164,7 @@
} else if (toolchain_args.current_cpu == "arm64") {
is_msvc_assembler = false
prefix = rebase_path("$clang_base_path/bin", root_build_dir)
- ml = "${goma_prefix}${prefix}/clang-cl.exe --target=arm64-windows"
+ ml = "${compiler_prefix}${prefix}/clang-cl.exe --target=arm64-windows"
x64 = ""
} else {
ml = "ml.exe"
@@ -175,14 +198,11 @@
}
tool("solink") {
- dllname = "{{root_out_dir}}/{{target_output_name}}{{output_extension}}" # e.g.
- # foo.dll
- libname = "{{root_out_dir}}/{{target_output_name}}{{output_extension}}.lib" # e.g.
- # foo.dll.lib
- expname = "{{root_out_dir}}/{{target_output_name}}{{output_extension}}.exp" # e.g.
- # foo.dll.exp
- pdbname = "{{root_out_dir}}/{{target_output_name}}{{output_extension}}.pdb" # e.g.
- # foo.dll.pdb
+ # E.g. "foo.dll":
+ dllname = "{{root_out_dir}}/{{target_output_name}}{{output_extension}}"
+ libname = "${dllname}.lib" # e.g. foo.dll.lib
+ expname = "${dllname}.exp"
+ pdbname = "${dllname}.pdb"
rspfile = "${dllname}.rsp"
link_command = "$python_path $tool_wrapper_path link-wrapper $env False link.exe /nologo /IMPLIB:$libname /DLL /OUT:$dllname /PDB:${dllname}.pdb @$rspfile"
@@ -253,21 +273,22 @@
assert(defined(invoker.toolchain_arch))
toolchain_arch = invoker.toolchain_arch
- msvc_toolchain(target_name) {
- environment = "environment." + toolchain_arch
- cl = "${goma_prefix}\"${vc_bin_dir}/cl.exe\""
- toolchain_args = {
- if (defined(invoker.toolchain_args)) {
- forward_variables_from(invoker.toolchain_args, "*")
- }
- current_cpu = toolchain_arch
- is_clang = false
- }
+ # The toolchain data for `msvc_toolchain()`.
+ if (toolchain_arch == "x86") {
+ win_toolchain_data = win_toolchain_data_x86
+ } else if (toolchain_arch == "x64") {
+ win_toolchain_data = win_toolchain_data_x64
+ } else if (toolchain_arch == "arm64") {
+ win_toolchain_data = win_toolchain_data_arm64
+ } else {
+ error("Unsupported toolchain_arch, add it to win_toolchain_data.gni")
}
+
msvc_toolchain("clang_" + target_name) {
environment = "environment." + toolchain_arch
prefix = rebase_path("$clang_base_path/bin", root_build_dir)
- cl = "${goma_prefix}$prefix/clang-cl.exe"
+ cl = "${compiler_prefix}$prefix/clang-cl.exe"
+ sys_include_flags = "${win_toolchain_data.include_flags_imsvc}"
toolchain_args = {
if (defined(invoker.toolchain_args)) {
forward_variables_from(invoker.toolchain_args, "*")
diff --git a/build/toolchain/win/setup_toolchain.py b/build/toolchain/win/setup_toolchain.py
index dbbee74..521c243 100644
--- a/build/toolchain/win/setup_toolchain.py
+++ b/build/toolchain/win/setup_toolchain.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Copyright 2013 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
@@ -10,69 +10,184 @@
# win tool. The script assumes that the root build directory is the current dir
# and the files will be written to the current directory.
+
import errno
+import json
import os
import re
import subprocess
import sys
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
+import gn_helpers
+
+SCRIPT_DIR = os.path.dirname(__file__)
+SDK_VERSION = '10.0.22621.0'
+
def _ExtractImportantEnvironment(output_of_set):
"""Extracts environment variables required for the toolchain to run from
a textual dump output by the cmd.exe 'set' command."""
envvars_to_save = (
+ 'cipd_cache_dir', # needed by vpython
+ 'homedrive', # needed by vpython
+ 'homepath', # needed by vpython
'goma_.*', # TODO(scottmg): This is ugly, but needed for goma.
'include',
'lib',
'libpath',
+ 'luci_context', # needed by vpython
'path',
'pathext',
'systemroot',
'temp',
'tmp',
+ 'userprofile', # needed by vpython
+ 'vpython_virtualenv_root' # needed by vpython
)
env = {}
+ # This occasionally happens and leads to misleading SYSTEMROOT error messages
+ # if not caught here.
+ if output_of_set.count('=') == 0:
+ raise Exception('Invalid output_of_set. Value is:\n%s' % output_of_set)
for line in output_of_set.splitlines():
for envvar in envvars_to_save:
- if re.match(envvar + '=', line.decode().lower()):
- var, setting = line.decode().split('=', 1)
+ if re.match(envvar + '=', line.lower()):
+ var, setting = line.split('=', 1)
if envvar == 'path':
- # Our own rules (for running gyp-win-tool) and other actions in
- # Chromium rely on python being in the path. Add the path to this
- # python here so that if it's not in the path when ninja is run
- # later, python will still be found.
+ # Our own rules and actions in Chromium rely on python being in the
+ # path. Add the path to this python here so that if it's not in the
+ # path when ninja is run later, python will still be found.
setting = os.path.dirname(sys.executable) + os.pathsep + setting
+ if envvar in ['include', 'lib']:
+ # Make sure that the include and lib paths point to directories that
+ # exist. This ensures a (relatively) clear error message if the
+ # required SDK is not installed.
+ for part in setting.split(';'):
+ if not os.path.exists(part) and len(part) != 0:
+ raise Exception(
+ 'Path "%s" from environment variable "%s" does not exist. '
+ 'Make sure the necessary SDK is installed.' % (part, envvar))
env[var.upper()] = setting
break
- for required in ('SYSTEMROOT', 'TEMP', 'TMP'):
- if required not in env:
- raise Exception('Environment variable "%s" '
- 'required to be set to valid path' % required)
+ if sys.platform in ('win32', 'cygwin'):
+ for required in ('SYSTEMROOT', 'TEMP', 'TMP'):
+ if required not in env:
+ raise Exception('Environment variable "%s" '
+ 'required to be set to valid path' % required)
return env
-def _SetupScript(target_cpu, sdk_dir):
- """Returns a command (with arguments) to be used to set up the
- environment."""
+def _DetectVisualStudioPath():
+ """Return path to the installed Visual Studio.
+ """
+
+ # Use the code in build/vs_toolchain.py to avoid duplicating code.
+ chromium_dir = os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..', '..'))
+ sys.path.append(os.path.join(chromium_dir, 'build'))
+ import vs_toolchain
+ return vs_toolchain.DetectVisualStudioPath()
+
+
+def _LoadEnvFromBat(args):
+ """Given a bat command, runs it and returns env vars set by it."""
+ args = args[:]
+ args.extend(('&&', 'set'))
+ popen = subprocess.Popen(
+ args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ variables, _ = popen.communicate()
+ if popen.returncode != 0:
+ raise Exception('"%s" failed with error %d' % (args, popen.returncode))
+ return variables.decode(errors='ignore')
+
+
+def _LoadToolchainEnv(cpu, toolchain_root, sdk_dir, target_store):
+ """Returns a dictionary with environment variables that must be set while
+ running binaries from the toolchain (e.g. INCLUDE and PATH for cl.exe)."""
# Check if we are running in the SDK command line environment and use
- # the setup script from the SDK if so.
- accepted_cpus = ('x86', 'x64', 'arm64')
- assert target_cpu in accepted_cpus, '%s not in accepted cpus %s' % (target_cpu, accepted_cpus)
+ # the setup script from the SDK if so. |cpu| should be either
+ # 'x86' or 'x64' or 'arm' or 'arm64'.
+ assert cpu in ('x86', 'x64', 'arm', 'arm64')
if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', 1))) and sdk_dir:
- return [os.path.normpath(os.path.join(sdk_dir, 'Bin/SetEnv.Cmd')),
- '/' + target_cpu]
+ # Load environment from json file.
+ env = os.path.normpath(os.path.join(sdk_dir, 'bin/SetEnv.%s.json' % cpu))
+ env = json.load(open(env))['env']
+ if env['VSINSTALLDIR'] == [["..", "..\\"]]:
+ # Old-style paths were relative to the win_sdk\bin directory.
+ json_relative_dir = os.path.join(sdk_dir, 'bin')
+ else:
+ # New-style paths are relative to the toolchain directory.
+ json_relative_dir = toolchain_root
+ for k in env:
+ entries = [os.path.join(*([json_relative_dir] + e)) for e in env[k]]
+ # clang-cl wants INCLUDE to be ;-separated even on non-Windows,
+ # lld-link wants LIB to be ;-separated even on non-Windows. Path gets :.
+ # The separator for INCLUDE here must match the one used in main() below.
+ sep = os.pathsep if k == 'PATH' else ';'
+ env[k] = sep.join(entries)
+ # PATH is a bit of a special case, it's in addition to the current PATH.
+ env['PATH'] = env['PATH'] + os.pathsep + os.environ['PATH']
+ # Augment with the current env to pick up TEMP and friends.
+ for k in os.environ:
+ if k not in env:
+ env[k] = os.environ[k]
+
+ varlines = []
+ for k in sorted(env.keys()):
+ varlines.append('%s=%s' % (str(k), str(env[k])))
+ variables = '\n'.join(varlines)
+
+ # Check that the json file contained the same environment as the .cmd file.
+ if sys.platform in ('win32', 'cygwin'):
+ script = os.path.normpath(os.path.join(sdk_dir, 'Bin/SetEnv.cmd'))
+ arg = '/' + cpu
+ json_env = _ExtractImportantEnvironment(variables)
+ cmd_env = _ExtractImportantEnvironment(_LoadEnvFromBat([script, arg]))
+ assert _LowercaseDict(json_env) == _LowercaseDict(cmd_env)
else:
- vcvars_arch = {
- 'x86': 'amd64_x86',
- 'x64': 'amd64',
- 'arm64': 'amd64_arm64',
- }
+ if 'GYP_MSVS_OVERRIDE_PATH' not in os.environ:
+ os.environ['GYP_MSVS_OVERRIDE_PATH'] = _DetectVisualStudioPath()
# We only support x64-hosted tools.
- # TODO(scottmg|dpranke): Non-depot_tools toolchain: need to get Visual
- # Studio install location from registry.
- return [os.path.normpath(os.path.join(os.environ['GYP_MSVS_OVERRIDE_PATH'],
- 'VC/Auxiliary/Build/vcvarsall.bat')),
- vcvars_arch[target_cpu]]
+ script_path = os.path.normpath(os.path.join(
+ os.environ['GYP_MSVS_OVERRIDE_PATH'],
+ 'VC/vcvarsall.bat'))
+ if not os.path.exists(script_path):
+ # vcvarsall.bat for VS 2017 fails if run after running vcvarsall.bat from
+ # VS 2013 or VS 2015. Fix this by clearing the vsinstalldir environment
+ # variable. Since vcvarsall.bat appends to the INCLUDE, LIB, and LIBPATH
+ # environment variables we need to clear those to avoid getting double
+ # entries when vcvarsall.bat has been run before gn gen. vcvarsall.bat
+ # also adds to PATH, but there is no clean way of clearing that and it
+ # doesn't seem to cause problems.
+ if 'VSINSTALLDIR' in os.environ:
+ del os.environ['VSINSTALLDIR']
+ if 'INCLUDE' in os.environ:
+ del os.environ['INCLUDE']
+ if 'LIB' in os.environ:
+ del os.environ['LIB']
+ if 'LIBPATH' in os.environ:
+ del os.environ['LIBPATH']
+ other_path = os.path.normpath(os.path.join(
+ os.environ['GYP_MSVS_OVERRIDE_PATH'],
+ 'VC/Auxiliary/Build/vcvarsall.bat'))
+ if not os.path.exists(other_path):
+ raise Exception('%s is missing - make sure VC++ tools are installed.' %
+ script_path)
+ script_path = other_path
+ cpu_arg = "amd64"
+ if (cpu != 'x64'):
+ # x64 is default target CPU thus any other CPU requires a target set
+ cpu_arg += '_' + cpu
+ args = [script_path, cpu_arg, ]
+ # Store target must come before any SDK version declaration
+ if (target_store):
+ args.append('store')
+ # Explicitly specifying the SDK version to build with to avoid accidentally
+ # building with a new and untested SDK. This should stay in sync with the
+ # packaged toolchain in build/vs_toolchain.py.
+ args.append(SDK_VERSION)
+ variables = _LoadEnvFromBat(args)
+ return _ExtractImportantEnvironment(variables)
def _FormatAsEnvironmentBlock(envvar_dict):
@@ -87,69 +202,126 @@
return block
-def _CopyTool(source_path):
- """Copies the given tool to the current directory, including a warning not
- to edit it."""
- with open(source_path) as source_file:
- tool_source = source_file.readlines()
+def _LowercaseDict(d):
+ """Returns a copy of `d` with both key and values lowercased.
- # Add header and write it out to the current directory (which should be the
- # root build dir).
- with open("gyp-win-tool", 'w') as tool_file:
- tool_file.write(''.join([tool_source[0],
- '# Generated by setup_toolchain.py do not edit.\n']
- + tool_source[1:]))
+ Args:
+ d: dict to lowercase (e.g. {'A': 'BcD'}).
+
+ Returns:
+ A dict with both keys and values lowercased (e.g.: {'a': 'bcd'}).
+ """
+ return {k.lower(): d[k].lower() for k in d}
+
+
+def FindFileInEnvList(env, env_name, separator, file_name, optional=False):
+ parts = env[env_name].split(separator)
+ for path in parts:
+ if os.path.exists(os.path.join(path, file_name)):
+ return os.path.realpath(path)
+ assert optional, "%s is not found in %s:\n%s\nCheck if it is installed." % (
+ file_name, env_name, '\n'.join(parts))
+ return ''
def main():
- if len(sys.argv) != 5:
+ if len(sys.argv) != 7:
print('Usage setup_toolchain.py '
'<visual studio path> <win sdk path> '
- '<runtime dirs> <target_cpu>')
+ '<runtime dirs> <target_os> <target_cpu> '
+ '<environment block name|none>')
sys.exit(2)
+ # toolchain_root and win_sdk_path are only read if the hermetic Windows
+ # toolchain is set, that is if DEPOT_TOOLS_WIN_TOOLCHAIN is not set to 0.
+ # With the hermetic Windows toolchain, the visual studio path in argv[1]
+ # is the root of the Windows toolchain directory.
+ toolchain_root = sys.argv[1]
win_sdk_path = sys.argv[2]
- runtime_dirs = sys.argv[3]
- target_cpu = sys.argv[4]
- cpus = ('x86', 'x64', 'arm64')
+ runtime_dirs = sys.argv[3]
+ target_os = sys.argv[4]
+ target_cpu = sys.argv[5]
+ environment_block_name = sys.argv[6]
+ if (environment_block_name == 'none'):
+ environment_block_name = ''
+
+ if (target_os == 'winuwp'):
+ target_store = True
+ else:
+ target_store = False
+
+ cpus = ('x86', 'x64', 'arm', 'arm64')
assert target_cpu in cpus
vc_bin_dir = ''
+ include = ''
+ lib = ''
# TODO(scottmg|goma): Do we need an equivalent of
# ninja_use_custom_environment_files?
+ def relflag(s): # Make s relative to builddir when cwd and sdk on same drive.
+ try:
+ return os.path.relpath(s).replace('\\', '/')
+ except ValueError:
+ return s
+
+ def q(s): # Quote s if it contains spaces or other weird characters.
+ return s if re.match(r'^[a-zA-Z0-9._/\\:-]*$', s) else '"' + s + '"'
+
for cpu in cpus:
- # Extract environment variables for subprocesses.
- args = _SetupScript(cpu, win_sdk_path)
- args.extend(('&&', 'set'))
- popen = subprocess.Popen(
- args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- variables, _ = popen.communicate()
- env = _ExtractImportantEnvironment(variables)
- env['PATH'] = runtime_dirs + ';' + env['PATH']
-
if cpu == target_cpu:
- for path in env['PATH'].split(os.pathsep):
- if os.path.exists(os.path.join(path, 'cl.exe')):
- vc_bin_dir = os.path.realpath(path)
- break
+ # Extract environment variables for subprocesses.
+ env = _LoadToolchainEnv(cpu, toolchain_root, win_sdk_path, target_store)
+ env['PATH'] = runtime_dirs + os.pathsep + env['PATH']
- # The Windows SDK include directories must be first. They both have a sal.h,
- # and the SDK one is newer and the SDK uses some newer features from it not
- # present in the Visual Studio one.
+ vc_bin_dir = FindFileInEnvList(env, 'PATH', os.pathsep, 'cl.exe')
- if win_sdk_path:
- additional_includes = ('{sdk_dir}\\Include\\shared;' +
- '{sdk_dir}\\Include\\um;' +
- '{sdk_dir}\\Include\\winrt;').format(
- sdk_dir=win_sdk_path)
- env['INCLUDE'] = additional_includes + env['INCLUDE']
- env_block = _FormatAsEnvironmentBlock(env)
- with open('environment.' + cpu, 'wb') as f:
- f.write(env_block.encode())
+ # The separator for INCLUDE here must match the one used in
+ # _LoadToolchainEnv() above.
+ include = [p.replace('"', r'\"') for p in env['INCLUDE'].split(';') if p]
+ include = list(map(relflag, include))
- assert vc_bin_dir
- print('vc_bin_dir = "%s"' % vc_bin_dir)
+ lib = [p.replace('"', r'\"') for p in env['LIB'].split(';') if p]
+ lib = list(map(relflag, lib))
+
+ include_I = ['/I' + i for i in include]
+ include_imsvc = ['-imsvc' + i for i in include]
+ libpath_flags = ['-libpath:' + i for i in lib]
+
+ if (environment_block_name != ''):
+ env_block = _FormatAsEnvironmentBlock(env)
+ with open(environment_block_name, 'w', encoding='utf8') as f:
+ f.write(env_block)
+
+ def ListToArgString(x):
+ return gn_helpers.ToGNString(' '.join(q(i) for i in x))
+
+ def ListToArgList(x):
+ return f'[{", ".join(gn_helpers.ToGNString(i) for i in x)}]'
+
+ print('vc_bin_dir = ' + gn_helpers.ToGNString(vc_bin_dir))
+ assert include_I
+ print(f'include_flags_I = {ListToArgString(include_I)}')
+ print(f'include_flags_I_list = {ListToArgList(include_I)}')
+ assert include_imsvc
+ if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', 1))) and win_sdk_path:
+ flags = ['/winsysroot' + relflag(toolchain_root)]
+ print(f'include_flags_imsvc = {ListToArgString(flags)}')
+ print(f'include_flags_imsvc_list = {ListToArgList(flags)}')
+ else:
+ print(f'include_flags_imsvc = {ListToArgString(include_imsvc)}')
+ print(f'include_flags_imsvc_list = {ListToArgList(include_imsvc)}')
+ print('paths = ' + gn_helpers.ToGNString(env['PATH']))
+ assert libpath_flags
+ print(f'libpath_flags = {ListToArgString(libpath_flags)}')
+ print(f'libpath_flags_list = {ListToArgList(libpath_flags)}')
+ if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', 1))) and win_sdk_path:
+ flags = ['/winsysroot:' + relflag(toolchain_root)]
+ print(f'libpath_lldlink_flags = {ListToArgString(flags)}')
+ print(f'libpath_lldlink_flags_list = {ListToArgList(flags)}')
+ else:
+ print(f'libpath_lldlink_flags = {ListToArgString(libpath_flags)}')
+ print(f'libpath_lldlink_flags_list = {ListToArgList(libpath_flags)}')
if __name__ == '__main__':
diff --git a/build/toolchain/win/win_toolchain_data.gni b/build/toolchain/win/win_toolchain_data.gni
new file mode 100644
index 0000000..a6e8328
--- /dev/null
+++ b/build/toolchain/win/win_toolchain_data.gni
@@ -0,0 +1,44 @@
+# Copyright 2023 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/config/win/visual_studio_version.gni")
+
+declare_args() {
+ win_toolchain_data_x86 =
+ exec_script("//build/toolchain/win/setup_toolchain.py",
+ [
+ visual_studio_path,
+ windows_sdk_path,
+ visual_studio_runtime_dirs,
+ "win",
+ "x86",
+ "environment.x86",
+ ],
+ "scope")
+
+ win_toolchain_data_x64 =
+ exec_script("//build/toolchain/win/setup_toolchain.py",
+ [
+ visual_studio_path,
+ windows_sdk_path,
+ visual_studio_runtime_dirs,
+ "win",
+ "x64",
+ "environment.x64",
+ ],
+ "scope")
+ if (target_cpu == "arm64" || host_cpu == "arm64") {
+ win_toolchain_data_arm64 =
+ exec_script("//build/toolchain/win/setup_toolchain.py",
+ [
+ visual_studio_path,
+ windows_sdk_path,
+ visual_studio_runtime_dirs,
+ "win",
+ "arm64",
+ "environment.arm64",
+ ],
+ "scope")
+ }
+}
diff --git a/build/vs_toolchain.py b/build/vs_toolchain.py
index 9625a1d..599b55a 100644
--- a/build/vs_toolchain.py
+++ b/build/vs_toolchain.py
@@ -30,7 +30,7 @@
from gn_helpers import ToGNString
script_dir = os.path.dirname(os.path.realpath(__file__))
-json_data_file = os.path.join(script_dir, 'win_toolchain.json')
+json_data_file = os.path.join(script_dir, 'new_win_toolchain.json')
sys.path.insert(0, os.path.join(script_dir))
@@ -496,6 +496,7 @@
'win_toolchain',
'get_toolchain_if_necessary.py'),
'--output-json', json_data_file,
+ '--toolchain-dir', os.path.join(depot_tools_path, 'win_toolchain'),
] + _GetDesiredVsToolchainHashes()
if force:
get_toolchain_args.append('--force')
