pkg/dartdev/lib/src/native_assets.dart

// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:io';

import 'package:code_assets/code_assets.dart';
import 'package:dartdev/src/commands/compile.dart' as compile;
import 'package:dartdev/src/native_assets_bundling.dart';
import 'package:dartdev/src/sdk.dart';
import 'package:dartdev/src/target.dart';
import 'package:dartdev/src/utils.dart';
import 'package:data_assets/data_assets.dart';
import 'package:file/local.dart';
import 'package:hooks/hooks.dart';
import 'package:hooks_runner/hooks_runner.dart'
    show
        BuildResult,
        LinkResult,
        NativeAssetsBuildRunner,
        PackageLayout,
        RecordUseConfig,
        UserDefines;
import 'package:logging/logging.dart';
import 'package:package_config/package_config.dart' as package_config;
import 'package:pub/pub.dart';

import 'core.dart';
import 'progress.dart';

class DartNativeAssetsBuilder {
  final Uri? pubspecUri;
  final Uri packageConfigUri;
  final package_config.PackageConfig packageConfig;
  final String runPackageName;
  final bool includeDevDependencies;
  final bool verbose;
  final bool dataAssetsExperimentEnabled;
  final compile.Sanitizer? sanitizer;

  static const _fileSystem = LocalFileSystem();

  late final Future<PackageLayout> _packageLayout = () async {
    return PackageLayout.fromPackageConfig(
      _fileSystem,
      packageConfig,
      packageConfigUri,
      runPackageName,
      includeDevDependencies: includeDevDependencies,
    );
  }();

  late final _logger = Logger('')
    ..onRecord.listen((LogRecord record) {
      final levelValue = record.level.value;
      if (levelValue >= Level.SEVERE.value) {
        log.stderr(record.message);
      } else if (levelValue >= Level.WARNING.value ||
          verbose && levelValue >= Level.INFO.value) {
        log.stdout(record.message);
      } else {
        // Note, this is ignored by default.
        log.trace(record.message);
      }
    });

  late final Future<NativeAssetsBuildRunner> _nativeAssetsBuildRunner =
      () async {
        return NativeAssetsBuildRunner(
          // This always runs in JIT mode.
          dartExecutable: Uri.file(sdk.dart),
          logger: _logger,
          fileSystem: const LocalFileSystem(),
          packageLayout: await _packageLayout,
          userDefines: UserDefines(workspacePubspec: pubspecUri),
        );
      }();

  DartNativeAssetsBuilder({
    this.pubspecUri,
    required this.packageConfigUri,
    required this.packageConfig,
    required this.runPackageName,
    required this.includeDevDependencies,
    required this.verbose,
    required this.dataAssetsExperimentEnabled,
    this.sanitizer,
    Target? target,
  }) : target = target ?? Target.current;

  /// Compiles all native assets for host OS in JIT mode.
  ///
  /// If provided, only native assets of all transitive dependencies of
  /// [runPackageName] are built.
  Future<List<EncodedAsset>?> compileNativeAssetsJit() async {
    final buildResult = await _buildNativeAssetsShared(linkingEnabled: false);
    if (buildResult == null) return null;
    return buildResult.encodedAssets;
  }

  /// Temporary directory where native assets are bundled for JIT execution.
  /// Deleted on VM shutdown via `--delete-temp-dir-on-shutdown`.
  Uri? tempDirUri;

  /// Compiles all native assets for host OS in JIT mode, and creates the
  /// native assets yaml file in a temporary directory.
  ///
  /// If provided, only native assets of all transitive dependencies of
  /// [runPackageName] are built.
  ///
  /// Used in `dart run` and `dart test`.
  Future<Uri?> compileNativeAssetsJitYamlFile() async {
    final assets = await compileNativeAssetsJit();
    if (assets == null) return null;

    final rawTempDir = await Directory.systemTemp.createTemp(
      'dart_native_assets_',
    );
    final tempDir = Directory(await rawTempDir.resolveSymbolicLinks());
    final outputUri = tempDir.uri.normalizePath();
    tempDirUri = outputUri;

    final kernelAssets = await bundleNativeAssets(
      assets,
      target,
      outputUri,
      relocatable: false,
    );

    final primaryYamlUri = await writeNativeAssetsYaml(
      kernelAssets,
      outputUri,
      header: '''# Native assets mapping for host OS in JIT mode.
# Generated by dartdev.
''',
    );

    final dartToolUri = packageConfigUri.resolve('.');
    // TODO(dacoharkes): Migrate all tools to read the unique native assets
    // YAML path instead of `.dart_tool/native_assets.yaml` so we can remove
    // this shared copy.
    final sharedYamlUri = dartToolUri.resolve('native_assets.yaml');
    try {
      await _atomicCopyFileIfChanged(
        File.fromUri(primaryYamlUri),
        File.fromUri(sharedYamlUri),
      );
    } catch (_) {
      // Best effort for tools that look for .dart_tool/native_assets.yaml.
    }

    return primaryYamlUri;
  }

  /// Atomically updates [target] with the contents of [source] only if the
  /// content has changed, avoiding non-atomic truncation races when concurrent
  /// processes read or write [target].
  Future<void> _atomicCopyFileIfChanged(File source, File target) async {
    if (await target.exists()) {
      try {
        if (await target.readAsString() == await source.readAsString()) {
          return;
        }
      } catch (_) {
        // If another process is concurrently writing or renaming [target],
        // reading may fail. Fall back to performing the copy.
      }
    }
    await target.parent.create(recursive: true);
    // Writing or copying directly to [target] non-atomically truncates the file,
    // which causes concurrent readers (e.g. kernel_service) to fail with EOF or
    // syntax errors. Writing to a temporary file on the same filesystem volume
    // and renaming it atomically replaces [target].
    final tempFile = File(
      '${target.path}.tmp.$pid.${DateTime.now().microsecondsSinceEpoch}',
    );
    await source.copy(tempFile.path);
    try {
      await tempFile.rename(target.path);
    } catch (_) {
      // On Windows, if another process holds an open read handle on [target],
      // or if another concurrent rename succeeded first, rename can fail.
      // Clean up the temporary file.
      try {
        await tempFile.delete();
      } catch (_) {}
    }
  }

  /// The packages with build hooks in the project.
  ///
  /// Fine to run together with the other methods, this is stateful, no work is
  /// duplicated.
  Future<List<String>> packagesWithBuildHooks() async {
    final builder = await _nativeAssetsBuildRunner;
    return await builder.packagesWithBuildHooks();
  }

  /// Check whether there are build hooks in the project.
  ///
  /// Fine to run together with the other methods, this is stateful, no work is
  /// duplicated.
  Future<bool> hasHooks() async {
    return (await packagesWithBuildHooks()).isNotEmpty;
  }

  Future<bool> warnOnNativeAssets() async {
    final builder = await _nativeAssetsBuildRunner;
    final packageNames = await builder.packagesWithBuildHooks();
    if (packageNames.isEmpty) return false;
    log.stderr(
      'Package(s) $packageNames require the native assets feature to be enabled. '
      'Enable native assets with `--enable-experiment=native-assets`.',
    );
    return true;
  }

  late final _extensions = [
    CodeAssetExtension(
      targetOS: target.os,
      linkModePreference: LinkModePreference.dynamic,
      targetArchitecture: target.architecture,
      macOS: _macOSConfig,
      cCompiler: _cCompilerConfig,
      sanitizer: switch (sanitizer) {
        null || compile.Sanitizer.none => null,
        compile.Sanitizer.asan => Sanitizer.asan,
        compile.Sanitizer.msan => Sanitizer.msan,
        compile.Sanitizer.tsan => Sanitizer.tsan,
      },
    ),
    if (dataAssetsExperimentEnabled) DataAssetsExtension(),
  ];

  Future<BuildResult?> _buildNativeAssetsShared({
    required bool linkingEnabled,
  }) async {
    final builder = await _nativeAssetsBuildRunner;
    final buildResult = await builder.build(
      extensions: _extensions,
      linkingEnabled: linkingEnabled,
    );
    if (buildResult.isFailure) return null;
    return buildResult.success;
  }

  Future<BuildResult?> buildNativeAssetsAOT() {
    return _buildNativeAssetsShared(linkingEnabled: true);
  }

  Future<LinkResult?> linkNativeAssetsAOT({
    required String? recordedUsagesPath,
    required List<Uri> entryPoints,
    required BuildResult buildResult,
  }) async {
    final builder = await _nativeAssetsBuildRunner;
    final linkResult = await builder.link(
      extensions: _extensions,
      recordUse: recordedUsagesPath != null
          ? RecordUseConfig(
              file: Uri.file(recordedUsagesPath),
              entryPoints: entryPoints,
              // The compiler inlines target operating system constants before Type
              // Flow Analysis (TFA) (see VMConstantEvaluator class and its
              // visitStaticGet method inside
              // pkg/vm/lib/transformations/vm_constant_evaluator.dart), which
              // alters the recorded usages (tree-shaking).
              // Thus, we must separate the link cache by target OS. Other build targets
              // (like architecture and sanitizers) do not affect this optimization.
              //
              // We also use a stable 'vm_aot' prefix rather than the dynamic SDK
              // compiler version string, ensuring old cache directories under
              // `.dart_tool/` are overwritten/re-used rather than accumulating
              // indefinitely across SDK rolls.
              compiler: 'vm_aot_${target.os.name}',
            )
          : null,
      buildResult: buildResult,
    );
    if (linkResult.isFailure) return null;
    return linkResult.success;
  }

  final Target target;

  late final _macOSConfig = target.os == OS.macOS
      ? MacOSCodeConfig(targetVersion: minimumSupportedMacOSVersion)
      : null;

  late final _cCompilerConfig = _getCCompilerConfig(target.os);

  CCompilerConfig? _getCCompilerConfig(OS targetOS) {
    // Specifically for running our tests on Dart CI with the test runner, we
    // recognize specific variables to setup the C Compiler configuration.
    final env = Platform.environment;
    final cc = env['DART_HOOK_TESTING_C_COMPILER__CC'];
    final ar = env['DART_HOOK_TESTING_C_COMPILER__AR'];
    final ld = env['DART_HOOK_TESTING_C_COMPILER__LD'];
    final envScript = env['DART_HOOK_TESTING_C_COMPILER__ENV_SCRIPT'];
    final envScriptArgs =
        env['DART_HOOK_TESTING_C_COMPILER__ENV_SCRIPT_ARGUMENTS']
            ?.split(' ')
            .map((arg) => arg.trim())
            .where((arg) => arg.isNotEmpty)
            .toList();
    if (cc != null && ar != null && ld != null) {
      return CCompilerConfig(
        archiver: Uri.file(ar),
        compiler: Uri.file(cc),
        linker: Uri.file(ld),
        windows: targetOS == OS.windows
            ? WindowsCCompilerConfig(
                developerCommandPrompt: envScript == null
                    ? null
                    : DeveloperCommandPrompt(
                        script: Uri.file(envScript),
                        arguments: envScriptArgs ?? [],
                      ),
              )
            : null,
      );
    }
    return null;
  }

  /// Runs `pub get` if no package config can be found.
  ///
  /// Returns `null` if no package config can be found, even after pub get.
  static Future<Uri?> ensurePackageConfig(Uri uri) async {
    var packageConfig = await _findPackageConfigUri(uri);
    // TODO(https://github.com/dart-lang/package_config/issues/126): Use
    // package config resolution from package:package_config.
    if (packageConfig == null) {
      final pubspecMaybe = await findPubspec(uri);
      if (pubspecMaybe != null) {
        // Silently run `pub get`, this is what would happen in
        // `getExecutableForCommand` later.
        try {
          await ensurePubspecResolved(
            pubspecMaybe.resolve('.').toFilePath(),
            progressGracePeriod: progressGracePeriod,
          );
        } on ResolutionFailedException catch (e) {
          log.stderr(e.message);
          return null;
        }
        packageConfig = await _findPackageConfigUri(uri);
      } else {
        return null;
      }
    }
    return packageConfig;
  }

  /// Tries to load the package config.
  ///
  /// Returns null and writes to stderr if the package config is malformed.
  static Future<package_config.PackageConfig?> loadPackageConfig(
    Uri packageConfigUri,
  ) async {
    try {
      return await package_config.loadPackageConfigUri(packageConfigUri);
    } on FormatException catch (e) {
      // This can be thrown if the package_config.json is malformed or has
      // duplicate entries.
      log.stderr(
        'Error encountered while parsing '
        '${packageConfigUri.toFilePath()}: ${e.message}.',
      );
      return null;
    }
  }

  /// Finds the package config uri.
  ///
  /// Returns `null` if no package config can be found.
  // TODO(https://github.com/dart-lang/package_config/issues/126): Expose this
  // logic in package:package_config.
  static Future<Uri?> _findPackageConfigUri(Uri uri) async {
    while (true) {
      final packageConfig = File.fromUri(
        uri.resolve('.dart_tool/package_config.json'),
      );
      final packageGraph = File.fromUri(
        uri.resolve('.dart_tool/package_graph.json'),
      );
      if (await packageConfig.exists() && await packageGraph.exists()) {
        return packageConfig.uri;
      }
      final parent = uri.resolve('..');
      if (parent == uri) {
        return null;
      }
      uri = parent;
    }
  }

  static Future<Uri?> findPubspec(Uri uri) async {
    while (true) {
      final candidate = uri.resolve('pubspec.yaml');
      if (await File.fromUri(candidate).exists()) {
        return candidate;
      }
      final parent = uri.resolve('..');
      if (parent == uri) {
        return null;
      }
      uri = parent;
    }
  }

  static Future<Uri?> findWorkspacePubspec(Uri? workspacePackageConfig) async {
    if (workspacePackageConfig == null) {
      return null;
    }
    final candidate = workspacePackageConfig.resolve('../pubspec.yaml');
    if (File.fromUri(candidate).existsSync()) {
      return candidate;
    }
    return null;
  }
}

/// Compiles the given [nativeAssetsYamlUri] to a kernel dill file at
/// [outputDillUri] using the frontend server.
Future<bool> compileNativeAssetsToDill({
  required Uri nativeAssetsYamlUri,
  required Uri outputDillUri,
}) async {
  if (!checkArtifactExists(sdk.frontendServerAotSnapshot) ||
      !checkArtifactExists(sdk.dartAotRuntime)) {
    return false;
  }
  final result = await Process.run(
    sdk.dartAotRuntime,
    [
      sdk.frontendServerAotSnapshot,
      '--native-assets-only',
      '--native-assets=${nativeAssetsYamlUri.toFilePath()}',
      '--output-dill=${outputDillUri.toFilePath()}',
    ],
  );
  if (result.exitCode != 0) {
    log.stderr(
      'Error: Compiling native assets to kernel failed:\n'
      '${result.stderr}\n${result.stdout}',
    );
    return false;
  }
  return true;
}

/// Compiles [nativeAssetsYamlUri] into a kernel component and concatenates it
/// with the kernel file at [kernelFilePath], returning a new concatenated dill
/// file path in [tempDirUri].
Future<String?> concatenateNativeAssetsKernel({
  required String kernelFilePath,
  required Uri nativeAssetsYamlUri,
  required Uri tempDirUri,
}) async {
  final tempNativeAssetsDill = tempDirUri.resolve('native_assets.dill');
  final success = await compileNativeAssetsToDill(
    nativeAssetsYamlUri: nativeAssetsYamlUri,
    outputDillUri: tempNativeAssetsDill,
  );
  if (!success) return null;

  final concatenatedDill = tempDirUri.resolve('concatenated.dill');
  final output = File.fromUri(concatenatedDill).openWrite();
  try {
    await output.addStream(File(kernelFilePath).openRead());
    await output.addStream(File.fromUri(tempNativeAssetsDill).openRead());
  } finally {
    await output.close();
  }

  return concatenatedDill.toFilePath();
}