pkg/dartdev/lib/src/executable_compiler.dart

// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'package:collection/collection.dart';
import 'package:package_config/package_config.dart';
import 'package:path/path.dart' as p;
import 'package:pub/pub.dart';

import 'core.dart';
import 'progress.dart';
import 'sdk.dart';

abstract final class ExecutableCompiler {
  /// Suffix appended to the snapshot path to get the depfile path.
  static const depfileExtension = '.d';

  /// Suffix appended to the snapshot path to get the metadata file path.
  ///
  /// The metadata file records the configuration the snapshot was compiled
  /// with, see [buildMetadata].
  static const metadataExtension = '.json';

  /// Version of the metadata file format, bumped when the contents change in
  /// a way older or newer SDKs cannot understand.
  static const metadataVersion = 1;

  /// Returns the expected snapshot path in `.dart_tool/dartdev/bin/`.
  ///
  /// The format matches
  /// `.dart_tool/dartdev/bin/<package>/<relativeScriptPath>.snapshot`.
  /// The SDK version and enabled experiments are stored in the companion
  /// metadata file (`<snapshot>.json`) rather than encoded in the filename, so
  /// that changing SDK or experiments overwrites the snapshot instead of
  /// leaving a stale one behind.
  static String pathOfSnapshot({
    required String workspaceRoot,
    required String packageName,
    required String relativeScriptPath,
  }) {
    return p.normalize(
      p.join(
        workspaceRoot,
        '.dart_tool',
        'dartdev',
        'bin',
        packageName,
        '$relativeScriptPath.snapshot',
      ),
    );
  }

  /// Returns the parent directory for temporary compilation directories in
  /// `.dart_tool/dartdev/tmp/`.
  ///
  /// Keeping temporary directories outside `.dart_tool/dartdev/bin/` ensures
  /// that [sweepStaleTempDirs] sweeps leftover directories across all packages
  /// and subdirectories while never colliding with package subdirectory names.
  static String pathOfTempDir({required String workspaceRoot}) {
    return p.normalize(p.join(workspaceRoot, '.dart_tool', 'dartdev', 'tmp'));
  }

  /// The version of the SDK that compiles the snapshots.
  static String get sdkVersion => Platform.version.split(' ').first;

  /// The configuration a snapshot was compiled with.
  ///
  /// A cached snapshot can only be reused if this matches the configuration of
  /// the current invocation, see [isSnapshotFresh].
  static Map<String, Object?> buildMetadata({
    required List<String> enabledExperiments,
  }) {
    return {
      'version': metadataVersion,
      'sdkVersion': sdkVersion,
      'enabledExperiments': [...enabledExperiments]..sort(),
    };
  }

  /// Parses a Ninja/Makefile-style depfile generated by the compiler.
  ///
  /// The format is:
  ///
  ///     target: dependency1 dependency2 ...
  ///
  /// Spaces in paths are escaped as `\ `, and backslashes as `\\`.
  /// Returns `null` if [content] does not contain the `: ` separator.
  static List<String>? parseDepfile(String content) {
    final colonIndex = content.indexOf(': ');
    if (colonIndex < 0) return null;

    final dependenciesString = content
        .substring(colonIndex + 1)
        .replaceAll(RegExp(r'[\r\n]'), '');
    final dependencies = <String>[];
    var index = 0;
    while (index < dependenciesString.length) {
      while (index < dependenciesString.length &&
          dependenciesString[index] == ' ') {
        index++;
      }
      if (index >= dependenciesString.length) break;
      final buffer = StringBuffer();
      while (index < dependenciesString.length) {
        final char = dependenciesString[index];
        if (char == ' ') {
          break;
        } else if (char == r'\') {
          index++;
          if (index >= dependenciesString.length) {
            // Malformed trailing backslash.
            return null;
          }
          buffer.write(dependenciesString[index]);
        } else {
          buffer.write(char);
        }
        index++;
      }
      if (buffer.isNotEmpty) {
        dependencies.add(buffer.toString());
      }
    }
    return dependencies;
  }

  /// Checks whether [snapshotFile] is up to date with respect to its inputs.
  ///
  /// Returns `true` if the snapshot exists, was compiled by this SDK with
  /// [enabledExperiments], is newer than [packageConfigFile], and is newer
  /// than all source files recorded in the depfile.
  static bool isSnapshotFresh({
    required File snapshotFile,
    required File packageConfigFile,
    required File depfile,
    required File metadataFile,
    required List<String> enabledExperiments,
  }) {
    try {
      if (!snapshotFile.existsSync() ||
          !packageConfigFile.existsSync() ||
          !depfile.existsSync() ||
          !metadataFile.existsSync()) {
        return false;
      }

      // The snapshot is only usable if it was built with the same compiler and
      // the same compilation options.
      final Object? metadata;
      try {
        metadata = jsonDecode(metadataFile.readAsStringSync());
      } on FormatException {
        return false;
      }
      final expectedMetadata = buildMetadata(
        enabledExperiments: enabledExperiments,
      );
      final actualExperiments = metadata is Map<String, Object?>
          ? metadata['enabledExperiments']
          : null;
      if (metadata is! Map<String, Object?> ||
          metadata['version'] != expectedMetadata['version'] ||
          metadata['sdkVersion'] != expectedMetadata['sdkVersion'] ||
          actualExperiments is! List ||
          !const ListEquality<Object?>().equals(
            actualExperiments,
            expectedMetadata['enabledExperiments'] as List<String>,
          )) {
        return false;
      }

      final snapshotModified = snapshotFile.lastModifiedSync();
      if (packageConfigFile.lastModifiedSync().isAfter(snapshotModified)) {
        return false;
      }

      final depfileContent = depfile.readAsStringSync();
      final dependencies = parseDepfile(depfileContent);
      if (dependencies == null) return false;

      for (final depPath in dependencies) {
        final depFile = File(depPath);
        if (!depFile.existsSync()) return false;
        if (depFile.lastModifiedSync().isAfter(snapshotModified)) return false;
      }

      return true;
    } on FileSystemException {
      return false;
    }
  }

  /// Compiles or returns the cached snapshot for [resolvedExecutable].
  ///
  /// The snapshot is compiled with [enabledExperiments] enabled, and a cached
  /// snapshot is only reused if it was compiled by this SDK with the same
  /// experiments, is newer than [packageConfigPath], and is newer than all
  /// source dependencies recorded in its depfile.
  ///
  /// If [verbose] is `true`, a `Built <package>:<executable>.` status line is
  /// written to stderr after a successful compilation.
  ///
  /// Returns a [DartExecutableWithPackageConfig] pointing to the snapshot file.
  static Future<DartExecutableWithPackageConfig> compile({
    required DartExecutableWithPackageConfig resolvedExecutable,
    List<String> enabledExperiments = const [],
    bool verbose = false,
  }) async {
    final executablePath = resolvedExecutable.executable;
    final packageConfigPath = resolvedExecutable.packageConfig;
    if (packageConfigPath == null) {
      // Direct file with no package config. Not snapshotted.
      return resolvedExecutable;
    }

    final packageConfigAbsolute = p.absolute(packageConfigPath);
    final packageConfig = await loadPackageConfigUri(
      p.toUri(packageConfigAbsolute),
    );

    final executableAbsolute = p.absolute(executablePath);
    final package = packageConfig.packageOf(p.toUri(executableAbsolute));
    if (package == null) {
      // Could not determine package. Return resolved executable as-is.
      return resolvedExecutable;
    }

    final packageRootPath = p.fromUri(package.root);
    if (!p.isWithin(packageRootPath, executableAbsolute)) {
      return resolvedExecutable;
    }

    final relativeScriptPath = p.relative(
      executableAbsolute,
      from: packageRootPath,
    );

    // Determine workspace root from .dart_tool/package_config.json
    final workspaceRoot = p.dirname(p.dirname(packageConfigAbsolute));
    final snapshotPath = pathOfSnapshot(
      workspaceRoot: workspaceRoot,
      packageName: package.name,
      relativeScriptPath: relativeScriptPath,
    );

    final snapshotFile = File(snapshotPath);
    final packageConfigFile = File(packageConfigAbsolute);
    final depfile = File('$snapshotPath$depfileExtension');
    final metadataFile = File('$snapshotPath$metadataExtension');

    if (!isSnapshotFresh(
      snapshotFile: snapshotFile,
      packageConfigFile: packageConfigFile,
      depfile: depfile,
      metadataFile: metadataFile,
      enabledExperiments: enabledExperiments,
    )) {
      final displayName =
          '${package.name}:${p.basenameWithoutExtension(executablePath)}';
      await _compileToSnapshot(
        executablePath: executableAbsolute,
        displayName: displayName,
        outputPath: snapshotPath,
        tempParentPath: pathOfTempDir(workspaceRoot: workspaceRoot),
        packageConfigPath: packageConfigAbsolute,
        enabledExperiments: enabledExperiments,
        verbose: verbose,
      );
    }

    return DartExecutableWithPackageConfig(
      executable: _relativeOrAbsolute(snapshotPath, p.current),
      packageConfig: packageConfigPath,
    );
  }

  /// Prefix of the temporary directories compilation output is written to.
  static const _tempDirPrefix = 'tmp';

  /// Age at which a leftover temporary directory is considered abandoned.
  ///
  /// Temporary directories are removed when compilation finishes, but they
  /// survive if the process is killed before that (for example with Ctrl+C).
  /// The threshold is generous so that a temporary directory belonging to a
  /// slow compilation in another process is never removed.
  static const defaultStaleTempDirAge = Duration(hours: 1);

  /// Removes temporary directories in [tempParentDir] that were left behind by
  /// compilations interrupted before they could clean up after themselves.
  ///
  /// A temporary directory is only removed if it has not been modified for
  /// [maxAge], so that directories in use by a concurrent compilation are
  /// left alone.
  ///
  /// This is best-effort: failures are ignored, as another process may be
  /// using the directory, or we may not be allowed to delete it.
  static void sweepStaleTempDirs(
    Directory tempParentDir, {
    Duration maxAge = defaultStaleTempDirAge,
  }) {
    final now = DateTime.now();
    final List<FileSystemEntity> entities;
    try {
      entities = tempParentDir.listSync(followLinks: false);
    } on FileSystemException {
      return;
    }
    for (final entity in entities) {
      if (entity is! Directory) continue;
      if (!p.basename(entity.path).startsWith(_tempDirPrefix)) continue;
      try {
        if (now.difference(entity.statSync().modified) < maxAge) continue;
        entity.deleteSync(recursive: true);
      } on FileSystemException {
        // Ignore.
      }
    }
  }

  static Future<void> _compileToSnapshot({
    required String executablePath,
    required String displayName,
    required String outputPath,
    required String tempParentPath,
    required String packageConfigPath,
    required List<String> enabledExperiments,
    required bool verbose,
  }) async {
    final outputDir = Directory(p.dirname(outputPath));
    if (!outputDir.existsSync()) {
      outputDir.createSync(recursive: true);
    }
    final tempParentDir = Directory(tempParentPath);
    if (!tempParentDir.existsSync()) {
      tempParentDir.createSync(recursive: true);
    }

    sweepStaleTempDirs(tempParentDir);
    final incrementalDillPath = '$outputPath.incremental';
    final tempDir = tempParentDir.createTempSync(_tempDirPrefix);
    final tempDill = p.join(tempDir.path, 'out.dill');
    final tempDepfile = '$tempDill$depfileExtension';
    final tempMetadataFile = '$tempDill$metadataExtension';
    final depfilePath = '$outputPath$depfileExtension';
    final metadataPath = '$outputPath$metadataExtension';

    // If an existing snapshot or incremental dill is present, copy it to a
    // temporary file and pass it as `--initialize-from-dill` to speed up
    // subsequent compilations without holding a file lock on the destination.
    final outputFile = File(outputPath);
    final depfile = File(depfilePath);
    final metadataFile = File(metadataPath);
    final incrementalFile = File(incrementalDillPath);
    String? initializeFromDill;
    final sourceDill = outputFile.existsSync()
        ? outputFile
        : (incrementalFile.existsSync() ? incrementalFile : null);
    if (sourceDill != null) {
      final tempInitDill = p.join(tempDir.path, 'init.dill');
      try {
        sourceDill.copySync(tempInitDill);
        initializeFromDill = tempInitDill;
      } on FileSystemException {
        // Ignore: fall back to compiling from scratch.
      }
    }

    try {
      final compilerSnapshot = sdk.frontendServerAotSnapshot;
      final aotRuntime = sdk.dartAotRuntime;
      final vmPlatformDill = sdk.vmPlatformDill;
      final sdkRoot = p.canonicalize(sdk.sdkPath);

      if (!checkArtifactExists(compilerSnapshot, logError: false) ||
          !checkArtifactExists(aotRuntime, logError: false) ||
          !checkArtifactExists(vmPlatformDill, logError: false)) {
        throw CompilationException(
          'Missing compiler snapshot, AOT runtime, or platform dill in Dart SDK. '
          'Have you built the full Dart SDK?',
        );
      }

      Future<void> runCompiler() async {
        final compileArgs = <String>[
          compilerSnapshot,
          '--sdk-root=$sdkRoot',
          '--platform=${p.toUri(vmPlatformDill)}',
          '--target=vm',
          '--packages=$packageConfigPath',
          for (final experiment in enabledExperiments)
            '--enable-experiment=$experiment',
          if (initializeFromDill != null)
            '--initialize-from-dill=$initializeFromDill',
          '--output-dill=$tempDill',
          '--depfile=$tempDepfile',
          '--incremental',
          '--no-print-incremental-dependencies',
          executablePath,
        ];

        final ProcessResult result;
        try {
          result = await Process.run(aotRuntime, compileArgs);
        } on ProcessException catch (e) {
          throw CompilationException(
            'Failed to start frontend compiler process ($aotRuntime): $e',
          );
        }

        if (result.exitCode == 0) {
          final tempFile = File(tempDill);
          if (!tempFile.existsSync() || !File(tempDepfile).existsSync()) {
            throw CompilationException(
              'Compilation did not produce expected output files at `$tempDill`.',
            );
          }
          File(tempMetadataFile).writeAsStringSync(
            jsonEncode(buildMetadata(enabledExperiments: enabledExperiments)),
          );
          // Delete any pre-existing metadata file before publishing the new
          // snapshot so that an interrupted rebuild cannot leave the new
          // snapshot paired with stale metadata.
          _tryDelete(metadataFile);
          await _renameFile(tempDill, outputPath);
          await _renameFile(tempDepfile, depfilePath);
          // Published last: without it the snapshot is considered stale, so a
          // crash in the middle of publishing cannot result in a snapshot
          // being reused with the wrong configuration.
          await _renameFile(tempMetadataFile, metadataPath);
          _tryDelete(incrementalFile);
        } else {
          final tempFile = File(tempDill);
          if (tempFile.existsSync()) {
            await _renameFile(tempDill, incrementalDillPath);
          }
          for (final file in [outputFile, depfile, metadataFile]) {
            _tryDelete(file);
          }
          final errorLines = _extractCompilerErrorMessages(
            '${result.stdout}\n${result.stderr}',
          );
          throw CompilationException(
            '${ansi.yellow}Failed to build $displayName:${ansi.none}\n$errorLines',
          );
        }
      }

      try {
        await progress(
          'Building package executable',
          runCompiler,
          transient: true,
          output: ProgressOutput.stderr,
        );
        if (verbose) {
          log.stderr('Built ${ansi.bold}$displayName${ansi.none}.');
        }
      } on CompilationException {
        rethrow;
      } catch (e) {
        throw CompilationException(
          '${ansi.yellow}Failed to build $displayName:${ansi.none}\n$e',
        );
      }
    } finally {
      if (tempDir.existsSync()) {
        try {
          tempDir.deleteSync(recursive: true);
        } on FileSystemException {
          // Ignore.
        }
      }
    }
  }

  static void _tryDelete(File file) {
    try {
      if (file.existsSync()) file.deleteSync();
    } on FileSystemException {
      // Ignore.
    }
  }

  static String _extractCompilerErrorMessages(String output) {
    String? boundaryKey;
    final lines = <String>[];
    for (final rawLine in LineSplitter.split(output)) {
      final trimmed = rawLine.trim();
      if (boundaryKey == null && trimmed.startsWith('result ')) {
        boundaryKey = trimmed.substring('result '.length).trim();
        continue;
      }
      if (boundaryKey != null &&
          (trimmed == boundaryKey || trimmed.startsWith('$boundaryKey '))) {
        continue;
      }
      lines.add(rawLine);
    }
    return lines.join('\n').trim();
  }

  static String _relativeOrAbsolute(String path, String from) {
    try {
      return p.normalize(p.relative(path, from: from));
    } on p.PathException {
      return p.normalize(path);
    }
  }

  /// Number of times replacing a locked file is retried on Windows.
  static const _windowsRenameAttempts = 10;

  static Future<void> _renameFile(String from, String to) async {
    final fromFile = File(from);
    if (!fromFile.existsSync()) return;
    final toFile = File(to);

    if (!Platform.isWindows) {
      fromFile.renameSync(to);
      return;
    }

    // On Windows the destination can be transiently locked (by antivirus, or
    // by another process executing the snapshot), so retry for a short while.
    for (var attempt = 0; attempt < _windowsRenameAttempts; attempt++) {
      try {
        _tryDelete(toFile);
        fromFile.renameSync(to);
        return;
      } on FileSystemException {
        await Future.delayed(const Duration(milliseconds: 20));
      }
    }

    // Still locked. Copying over the destination is more likely to succeed
    // than replacing it, as it does not require removing the locked file.
    fromFile.copySync(to);
    _tryDelete(fromFile);
  }
}

final class CompilationException implements Exception {
  final String message;
  CompilationException(this.message);

  @override
  String toString() => message;
}