blob: 2b6e9b93324a4a31ae68bf24cdef3f31746299f6 [file] [edit]
// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:collection/collection.dart';
import 'package:package_config/package_config.dart';
import 'package:path/path.dart' as p;
import 'package:pub/pub.dart';
import 'core.dart';
import 'progress.dart';
import 'sdk.dart';
abstract final class ExecutableCompiler {
/// Suffix appended to the snapshot path to get the depfile path.
static const depfileExtension = '.d';
/// Suffix appended to the snapshot path to get the metadata file path.
///
/// The metadata file records the configuration the snapshot was compiled
/// with, see [buildMetadata].
static const metadataExtension = '.json';
/// Version of the metadata file format, bumped when the contents change in
/// a way older or newer SDKs cannot understand.
static const metadataVersion = 1;
/// Returns the expected snapshot path in `.dart_tool/dartdev/bin/`.
///
/// The format matches
/// `.dart_tool/dartdev/bin/<package>/<relativeScriptPath>.snapshot`.
/// The SDK version and enabled experiments are stored in the companion
/// metadata file (`<snapshot>.json`) rather than encoded in the filename, so
/// that changing SDK or experiments overwrites the snapshot instead of
/// leaving a stale one behind.
static String pathOfSnapshot({
required String workspaceRoot,
required String packageName,
required String relativeScriptPath,
}) {
return p.normalize(
p.join(
workspaceRoot,
'.dart_tool',
'dartdev',
'bin',
packageName,
'$relativeScriptPath.snapshot',
),
);
}
/// Returns the parent directory for temporary compilation directories in
/// `.dart_tool/dartdev/tmp/`.
///
/// Keeping temporary directories outside `.dart_tool/dartdev/bin/` ensures
/// that [sweepStaleTempDirs] sweeps leftover directories across all packages
/// and subdirectories while never colliding with package subdirectory names.
static String pathOfTempDir({required String workspaceRoot}) {
return p.normalize(p.join(workspaceRoot, '.dart_tool', 'dartdev', 'tmp'));
}
/// The version of the SDK that compiles the snapshots.
static String get sdkVersion => Platform.version.split(' ').first;
/// The configuration a snapshot was compiled with.
///
/// A cached snapshot can only be reused if this matches the configuration of
/// the current invocation, see [isSnapshotFresh].
static Map<String, Object?> buildMetadata({
required List<String> enabledExperiments,
}) {
return {
'version': metadataVersion,
'sdkVersion': sdkVersion,
'enabledExperiments': [...enabledExperiments]..sort(),
};
}
/// Parses a Ninja/Makefile-style depfile generated by the compiler.
///
/// The format is:
///
/// target: dependency1 dependency2 ...
///
/// Spaces in paths are escaped as `\ `, and backslashes as `\\`.
/// Returns `null` if [content] does not contain the `: ` separator.
static List<String>? parseDepfile(String content) {
final colonIndex = content.indexOf(': ');
if (colonIndex < 0) return null;
final dependenciesString = content
.substring(colonIndex + 1)
.replaceAll(RegExp(r'[\r\n]'), '');
final dependencies = <String>[];
var index = 0;
while (index < dependenciesString.length) {
while (index < dependenciesString.length &&
dependenciesString[index] == ' ') {
index++;
}
if (index >= dependenciesString.length) break;
final buffer = StringBuffer();
while (index < dependenciesString.length) {
final char = dependenciesString[index];
if (char == ' ') {
break;
} else if (char == r'\') {
index++;
if (index >= dependenciesString.length) {
// Malformed trailing backslash.
return null;
}
buffer.write(dependenciesString[index]);
} else {
buffer.write(char);
}
index++;
}
if (buffer.isNotEmpty) {
dependencies.add(buffer.toString());
}
}
return dependencies;
}
/// Checks whether [snapshotFile] is up to date with respect to its inputs.
///
/// Returns `true` if the snapshot exists, was compiled by this SDK with
/// [enabledExperiments], is newer than [packageConfigFile], and is newer
/// than all source files recorded in the depfile.
static bool isSnapshotFresh({
required File snapshotFile,
required File packageConfigFile,
required File depfile,
required File metadataFile,
required List<String> enabledExperiments,
}) {
try {
if (!snapshotFile.existsSync() ||
!packageConfigFile.existsSync() ||
!depfile.existsSync() ||
!metadataFile.existsSync()) {
return false;
}
// The snapshot is only usable if it was built with the same compiler and
// the same compilation options.
final Object? metadata;
try {
metadata = jsonDecode(metadataFile.readAsStringSync());
} on FormatException {
return false;
}
final expectedMetadata = buildMetadata(
enabledExperiments: enabledExperiments,
);
final actualExperiments = metadata is Map<String, Object?>
? metadata['enabledExperiments']
: null;
if (metadata is! Map<String, Object?> ||
metadata['version'] != expectedMetadata['version'] ||
metadata['sdkVersion'] != expectedMetadata['sdkVersion'] ||
actualExperiments is! List ||
!const ListEquality<Object?>().equals(
actualExperiments,
expectedMetadata['enabledExperiments'] as List<String>,
)) {
return false;
}
final snapshotModified = snapshotFile.lastModifiedSync();
if (packageConfigFile.lastModifiedSync().isAfter(snapshotModified)) {
return false;
}
final depfileContent = depfile.readAsStringSync();
final dependencies = parseDepfile(depfileContent);
if (dependencies == null) return false;
for (final depPath in dependencies) {
final depFile = File(depPath);
if (!depFile.existsSync()) return false;
if (depFile.lastModifiedSync().isAfter(snapshotModified)) return false;
}
return true;
} on FileSystemException {
return false;
}
}
/// Compiles or returns the cached snapshot for [resolvedExecutable].
///
/// The snapshot is compiled with [enabledExperiments] enabled, and a cached
/// snapshot is only reused if it was compiled by this SDK with the same
/// experiments, is newer than [packageConfigPath], and is newer than all
/// source dependencies recorded in its depfile.
///
/// Returns a [DartExecutableWithPackageConfig] pointing to the snapshot file.
static Future<DartExecutableWithPackageConfig> compile({
required DartExecutableWithPackageConfig resolvedExecutable,
List<String> enabledExperiments = const [],
bool quiet = false,
}) async {
final executablePath = resolvedExecutable.executable;
final packageConfigPath = resolvedExecutable.packageConfig;
if (packageConfigPath == null) {
// Direct file with no package config. Not snapshotted.
return resolvedExecutable;
}
final packageConfigAbsolute = p.absolute(packageConfigPath);
final packageConfig = await loadPackageConfigUri(
p.toUri(packageConfigAbsolute),
);
final executableAbsolute = p.absolute(executablePath);
final package = packageConfig.packageOf(p.toUri(executableAbsolute));
if (package == null) {
// Could not determine package. Return resolved executable as-is.
return resolvedExecutable;
}
final packageRootPath = p.fromUri(package.root);
if (!p.isWithin(packageRootPath, executableAbsolute)) {
return resolvedExecutable;
}
final relativeScriptPath = p.relative(
executableAbsolute,
from: packageRootPath,
);
// Determine workspace root from .dart_tool/package_config.json
final workspaceRoot = p.dirname(p.dirname(packageConfigAbsolute));
final snapshotPath = pathOfSnapshot(
workspaceRoot: workspaceRoot,
packageName: package.name,
relativeScriptPath: relativeScriptPath,
);
final snapshotFile = File(snapshotPath);
final packageConfigFile = File(packageConfigAbsolute);
final depfile = File('$snapshotPath$depfileExtension');
final metadataFile = File('$snapshotPath$metadataExtension');
if (!isSnapshotFresh(
snapshotFile: snapshotFile,
packageConfigFile: packageConfigFile,
depfile: depfile,
metadataFile: metadataFile,
enabledExperiments: enabledExperiments,
)) {
final displayName =
'${package.name}:${p.basenameWithoutExtension(executablePath)}';
await _compileToSnapshot(
executablePath: executableAbsolute,
displayName: displayName,
outputPath: snapshotPath,
tempParentPath: pathOfTempDir(workspaceRoot: workspaceRoot),
packageConfigPath: packageConfigAbsolute,
enabledExperiments: enabledExperiments,
quiet: quiet,
);
}
return DartExecutableWithPackageConfig(
executable: _relativeOrAbsolute(snapshotPath, p.current),
packageConfig: packageConfigPath,
);
}
/// Prefix of the temporary directories compilation output is written to.
static const _tempDirPrefix = 'tmp';
/// Age at which a leftover temporary directory is considered abandoned.
///
/// Temporary directories are removed when compilation finishes, but they
/// survive if the process is killed before that (for example with Ctrl+C).
/// The threshold is generous so that a temporary directory belonging to a
/// slow compilation in another process is never removed.
static const defaultStaleTempDirAge = Duration(hours: 1);
/// Removes temporary directories in [tempParentDir] that were left behind by
/// compilations interrupted before they could clean up after themselves.
///
/// A temporary directory is only removed if it has not been modified for
/// [maxAge], so that directories in use by a concurrent compilation are
/// left alone.
///
/// This is best-effort: failures are ignored, as another process may be
/// using the directory, or we may not be allowed to delete it.
static void sweepStaleTempDirs(
Directory tempParentDir, {
Duration maxAge = defaultStaleTempDirAge,
}) {
final now = DateTime.now();
final List<FileSystemEntity> entities;
try {
entities = tempParentDir.listSync(followLinks: false);
} on FileSystemException {
return;
}
for (final entity in entities) {
if (entity is! Directory) continue;
if (!p.basename(entity.path).startsWith(_tempDirPrefix)) continue;
try {
if (now.difference(entity.statSync().modified) < maxAge) continue;
entity.deleteSync(recursive: true);
} on FileSystemException {
// Ignore.
}
}
}
static Future<void> _compileToSnapshot({
required String executablePath,
required String displayName,
required String outputPath,
required String tempParentPath,
required String packageConfigPath,
required List<String> enabledExperiments,
bool quiet = false,
}) async {
final outputDir = Directory(p.dirname(outputPath));
if (!outputDir.existsSync()) {
outputDir.createSync(recursive: true);
}
final tempParentDir = Directory(tempParentPath);
if (!tempParentDir.existsSync()) {
tempParentDir.createSync(recursive: true);
}
sweepStaleTempDirs(tempParentDir);
final incrementalDillPath = '$outputPath.incremental';
final tempDir = tempParentDir.createTempSync(_tempDirPrefix);
final tempDill = p.join(tempDir.path, 'out.dill');
final tempDepfile = '$tempDill$depfileExtension';
final tempMetadataFile = '$tempDill$metadataExtension';
final depfilePath = '$outputPath$depfileExtension';
final metadataPath = '$outputPath$metadataExtension';
// If an existing snapshot or incremental dill is present, copy it to a
// temporary file and pass it as `--initialize-from-dill` to speed up
// subsequent compilations without holding a file lock on the destination.
final outputFile = File(outputPath);
final depfile = File(depfilePath);
final metadataFile = File(metadataPath);
final incrementalFile = File(incrementalDillPath);
String? initializeFromDill;
final sourceDill = outputFile.existsSync()
? outputFile
: (incrementalFile.existsSync() ? incrementalFile : null);
if (sourceDill != null) {
final tempInitDill = p.join(tempDir.path, 'init.dill');
try {
sourceDill.copySync(tempInitDill);
initializeFromDill = tempInitDill;
} on FileSystemException {
// Ignore: fall back to compiling from scratch.
}
}
try {
final compilerSnapshot = sdk.frontendServerAotSnapshot;
final aotRuntime = sdk.dartAotRuntime;
final vmPlatformDill = sdk.vmPlatformDill;
final sdkRoot = p.canonicalize(sdk.sdkPath);
if (!checkArtifactExists(compilerSnapshot, logError: false) ||
!checkArtifactExists(aotRuntime, logError: false) ||
!checkArtifactExists(vmPlatformDill, logError: false)) {
throw CompilationException(
'Missing compiler snapshot, AOT runtime, or platform dill in Dart SDK. '
'Have you built the full Dart SDK?',
);
}
Future<void> runCompiler() async {
final compileArgs = <String>[
compilerSnapshot,
'--sdk-root=$sdkRoot',
'--platform=${p.toUri(vmPlatformDill)}',
'--target=vm',
'--packages=$packageConfigPath',
for (final experiment in enabledExperiments)
'--enable-experiment=$experiment',
if (initializeFromDill != null)
'--initialize-from-dill=$initializeFromDill',
'--output-dill=$tempDill',
'--depfile=$tempDepfile',
'--incremental',
'--no-print-incremental-dependencies',
executablePath,
];
final ProcessResult result;
try {
result = await Process.run(aotRuntime, compileArgs);
} on ProcessException catch (e) {
throw CompilationException(
'Failed to start frontend compiler process ($aotRuntime): $e',
);
}
if (result.exitCode == 0) {
final tempFile = File(tempDill);
if (!tempFile.existsSync() || !File(tempDepfile).existsSync()) {
throw CompilationException(
'Compilation did not produce expected output files at `$tempDill`.',
);
}
File(tempMetadataFile).writeAsStringSync(
jsonEncode(buildMetadata(enabledExperiments: enabledExperiments)),
);
// Delete any pre-existing metadata file before publishing the new
// snapshot so that an interrupted rebuild cannot leave the new
// snapshot paired with stale metadata.
_tryDelete(metadataFile);
await _renameFile(tempDill, outputPath);
await _renameFile(tempDepfile, depfilePath);
// Published last: without it the snapshot is considered stale, so a
// crash in the middle of publishing cannot result in a snapshot
// being reused with the wrong configuration.
await _renameFile(tempMetadataFile, metadataPath);
_tryDelete(incrementalFile);
} else {
final tempFile = File(tempDill);
if (tempFile.existsSync()) {
await _renameFile(tempDill, incrementalDillPath);
}
for (final file in [outputFile, depfile, metadataFile]) {
_tryDelete(file);
}
final errorLines = _extractCompilerErrorMessages(
'${result.stdout}\n${result.stderr}',
);
throw CompilationException(
'${ansi.yellow}Failed to build $displayName:${ansi.none}\n$errorLines',
);
}
}
try {
if (!quiet && stderr.hasTerminal) {
await progress(
'Building package executable',
runCompiler,
progressUpdatesOnStderr: true,
);
log.stderr('Built ${ansi.bold}$displayName${ansi.none}.');
} else {
await runCompiler();
}
} on CompilationException {
rethrow;
} catch (e) {
throw CompilationException(
'${ansi.yellow}Failed to build $displayName:${ansi.none}\n$e',
);
}
} finally {
if (tempDir.existsSync()) {
try {
tempDir.deleteSync(recursive: true);
} on FileSystemException {
// Ignore.
}
}
}
}
static void _tryDelete(File file) {
try {
if (file.existsSync()) file.deleteSync();
} on FileSystemException {
// Ignore.
}
}
static String _extractCompilerErrorMessages(String output) {
String? boundaryKey;
final lines = <String>[];
for (final rawLine in LineSplitter.split(output)) {
final trimmed = rawLine.trim();
if (boundaryKey == null && trimmed.startsWith('result ')) {
boundaryKey = trimmed.substring('result '.length).trim();
continue;
}
if (boundaryKey != null &&
(trimmed == boundaryKey || trimmed.startsWith('$boundaryKey '))) {
continue;
}
lines.add(rawLine);
}
return lines.join('\n').trim();
}
static String _relativeOrAbsolute(String path, String from) {
try {
return p.normalize(p.relative(path, from: from));
} on p.PathException {
return p.normalize(path);
}
}
/// Number of times replacing a locked file is retried on Windows.
static const _windowsRenameAttempts = 10;
static Future<void> _renameFile(String from, String to) async {
final fromFile = File(from);
if (!fromFile.existsSync()) return;
final toFile = File(to);
if (!Platform.isWindows) {
fromFile.renameSync(to);
return;
}
// On Windows the destination can be transiently locked (by antivirus, or
// by another process executing the snapshot), so retry for a short while.
for (var attempt = 0; attempt < _windowsRenameAttempts; attempt++) {
try {
_tryDelete(toFile);
fromFile.renameSync(to);
return;
} on FileSystemException {
await Future.delayed(const Duration(milliseconds: 20));
}
}
// Still locked. Copying over the destination is more likely to succeed
// than replacing it, as it does not require removing the locked file.
fromFile.copySync(to);
_tryDelete(fromFile);
}
}
final class CompilationException implements Exception {
final String message;
CompilationException(this.message);
@override
String toString() => message;
}