pkg/front_end/tool/entry_points.dart

// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:convert' show JsonEncoder, LineSplitter, jsonDecode, utf8;
import 'dart:io'
    show File, Platform, ProcessSignal, exit, exitCode, stderr, stdin, stdout;

import 'package:_fe_analyzer_shared/src/util/relativize.dart'
    show isWindows, relativizeUri;
import 'package:front_end/src/api_prototype/compiler_options.dart'
    show CompilerOptions, CfeDiagnosticMessage;
import 'package:front_end/src/api_prototype/incremental_kernel_generator.dart';
import 'package:front_end/src/api_prototype/kernel_generator.dart';
import 'package:front_end/src/base/command_line_options.dart';
import 'package:front_end/src/base/compiler_context.dart' show CompilerContext;
import 'package:front_end/src/base/file_system_dependency_tracker.dart';
import 'package:front_end/src/base/get_dependencies.dart' show getDependencies;
import 'package:front_end/src/base/incremental_compiler.dart'
    show IncrementalCompiler;
import 'package:front_end/src/base/processed_options.dart'
    show ProcessedOptions;

import 'package:front_end/src/codes/diagnostic.dart' as diag;
import 'package:front_end/src/kernel/benchmarker.dart'
    show BenchmarkPhases, Benchmarker;
import 'package:front_end/src/kernel/utils.dart' show writeComponentToFile;
import 'package:front_end/src/kernel_generator_impl.dart'
    show generateKernelInternal;
import 'package:front_end/src/linux_and_intel_specific_perf.dart';
import 'package:kernel/kernel.dart'
    show Component, Library, RecursiveVisitor, Source;
import 'package:kernel/src/types.dart' show Types;
import 'package:kernel/target/targets.dart' show Target, TargetFlags, getTarget;
import 'package:kernel/verifier.dart';

import '../test/coverage_helper.dart';
import 'additional_targets.dart' show installAdditionalTargets;
import 'bench_maker.dart' show BenchMaker;
import 'command_line.dart' show runProtectedFromAbort, withGlobalOptions;

const bool benchmark = const bool.fromEnvironment(
  "benchmark",
  defaultValue: false,
);

const bool summary =
    const bool.fromEnvironment("summary", defaultValue: false) || benchmark;

const int iterations = const int.fromEnvironment("iterations", defaultValue: 1);

Future<void> compileEntryPoint(List<String> arguments) async {
  if (Platform.environment["dart_cfe_intel_pt"] == "true") {
    print("Notice: Locating perf for profiling using intel_pt events.");
    linuxAndIntelSpecificPerf(onlyInitialize: true);
  }
  bool useIncrementalCompiler = Platform.environment["incremental"] == "true";
  installAdditionalTargets();

  // Timing results for each iteration
  List<double> elapsedTimes = <double>[];
  List<Benchmarker> benchmarkers = <Benchmarker>[];

  for (int i = 0; i < iterations; i++) {
    if (i > 0) {
      print("\n\n=== Iteration ${i + 1} of $iterations");
    }
    Stopwatch stopwatch = new Stopwatch()..start();
    Benchmarker? benchmarker;
    if (benchmark) {
      benchmarker = new Benchmarker();
      benchmarkers.add(benchmarker);
    }
    await compile(
      arguments,
      benchmarker: benchmarker,
      useIncrementalCompiler: useIncrementalCompiler,
    );
    benchmarker?.stop();
    stopwatch.stop();

    elapsedTimes.add(stopwatch.elapsedMilliseconds.toDouble());
    List<Object>? typeChecks = Types.typeChecksForTesting;
    if (typeChecks?.isNotEmpty ?? false) {
      BenchMaker.writeTypeChecks("type_checks.json", typeChecks!);
    }
  }

  summarize(elapsedTimes, benchmarkers);
}

Future<void> outlineEntryPoint(List<String> arguments) async {
  installAdditionalTargets();

  // Timing results for each iteration
  List<double> elapsedTimes = <double>[];
  List<Benchmarker> benchmarkers = <Benchmarker>[];

  for (int i = 0; i < iterations; i++) {
    if (i > 0) {
      print("\n\n=== Iteration ${i + 1} of $iterations");
    }
    Stopwatch stopwatch = new Stopwatch()..start();
    Benchmarker? benchmarker;
    if (benchmark) {
      benchmarker = new Benchmarker();
      benchmarkers.add(benchmarker);
    }
    await outline(arguments, benchmarker: benchmarker);
    benchmarker?.stop();
    stopwatch.stop();

    elapsedTimes.add(stopwatch.elapsedMilliseconds.toDouble());
  }

  summarize(elapsedTimes, benchmarkers);
}

void summarize(List<double> elapsedTimes, List<Benchmarker> benchmarkers) {
  if (summary) {
    Map<String, dynamic> map = <String, dynamic>{
      'elapsedTimes': elapsedTimes,
      if (benchmarkers.isNotEmpty) 'benchmarkers': benchmarkers,
    };
    JsonEncoder encoder = new JsonEncoder.withIndent("  ");
    String json = encoder.convert(map);
    print('\nSummary:\n\n$json\n');
  } else {
    assert(benchmarkers.isEmpty);
  }
}

Future<void> depsEntryPoint(List<String> arguments) async {
  installAdditionalTargets();

  for (int i = 0; i < iterations; i++) {
    if (i > 1) {
      print("\n");
    }
    await deps(arguments);
  }
}

Future<void> compilePlatformEntryPoint(List<String> arguments) async {
  installAdditionalTargets();
  for (int i = 0; i < iterations; i++) {
    if (i > 0) {
      print("\n");
    }
    await runProtectedFromAbort<void>(() => compilePlatform(arguments));
  }
}

Future<void> batchEntryPoint(List<String> arguments) async {
  if (shouldCollectCoverage()) {
    tryListenToSignal(
      ProcessSignal.sigterm,
      () => possiblyCollectCoverage("batch_compiler", doExit: true),
    );
  }
  installAdditionalTargets();
  await new BatchCompiler(
    stdin.transform(utf8.decoder).transform(new LineSplitter()),
  ).run();
}

void tryListenToSignal(ProcessSignal signal, void Function() callback) {
  try {
    signal.watch().listen(
      (_) => callback(),
      onError: (_) {
        // swallow.
      },
    );
  } catch (e) {
    // swallow.
  }
}

const String cfeCoverageEnvironmentVariable = "CFE_COVERAGE";

bool shouldCollectCoverage() {
  String? coverage = Platform.environment[cfeCoverageEnvironmentVariable];
  if (coverage != null) return true;
  return false;
}

Future<void> possiblyCollectCoverage(
  String displayNamePrefix, {
  required bool doExit,
}) async {
  String? coverage = Platform.environment[cfeCoverageEnvironmentVariable];

  if (coverage != null) {
    assert(shouldCollectCoverage());
    Uri coverageUri = Uri.base.resolveUri(Uri.file(coverage));
    String displayName =
        "${displayNamePrefix}_${DateTime.now().microsecondsSinceEpoch}";
    File f = new File.fromUri(coverageUri.resolve("$displayName.coverage"));
    // Force compiling seems to add something like 1 second to the collection
    // time, but we get rid of uncompiled functions so it seems to be worth it.
    (await collectCoverage(
      displayName: displayName,
      forceCompile: true,
    ))?.writeToFile(f);
  }

  if (doExit) {
    exit(exitCode);
  }
}

class BatchCompiler {
  final Stream<String>? lines;

  Uri? platformUri;

  Component? platformComponent;

  bool hadVerifyError = false;

  IncrementalCompiler? _incrementalCompiler;

  List<CfeDiagnosticMessage> _errors = [];

  void Function(CfeDiagnosticMessage)? _originalOnDiagnostic;

  new(this.lines);

  Future<void> run() async {
    await for (String line in lines!) {
      try {
        if (await batchCompileArguments(
          new List<String>.from(jsonDecode(line)),
        )) {
          stdout.writeln(">>> TEST OK");
        } else {
          stdout.writeln(">>> TEST FAIL");
        }
      } catch (e, trace) {
        stderr.writeln("Unhandled exception:\n  $e");
        stderr.writeln(trace);
        stdout.writeln(">>> TEST CRASH");
        _incrementalCompiler = null;
      }
      await stdout.flush();
      stderr.writeln(">>> EOF STDERR");
      await stderr.flush();
    }
  }

  Future<bool> batchCompileArguments(List<String> arguments) {
    return runProtectedFromAbort<bool>(
      () => withGlobalOptions<bool>(
        "compile",
        [Flags.omitPlatform, ...arguments],
        true,
        (CompilerContext c, _) => batchCompileImpl(c),
      ),
      false,
    );
  }

  Future<bool> batchCompile(CompilerOptions options, Uri input, Uri output) {
    return CompilerContext.runWithOptions(
      new ProcessedOptions(
        options: options,
        inputs: <Uri>[input],
        output: output,
      ),
      batchCompileImpl,
    );
  }

  void _onDiagnostic(CfeDiagnosticMessage message) {
    _errors.add(message);
    if (_originalOnDiagnostic != null) {
      _originalOnDiagnostic!(message);
    }
  }

  Future<bool> batchCompileImpl(CompilerContext context) async {
    _errors.clear();
    ProcessedOptions options = context.options;
    bool createNewCompiler = false;
    if (_incrementalCompiler == null ||
        !_incrementalCompiler!.context.options.equivalent(options)) {
      createNewCompiler = true;
    }

    if (platformComponent == null ||
        platformUri != options.sdkSummary ||
        hadVerifyError) {
      createNewCompiler = true;
      platformUri = options.sdkSummary;
      platformComponent = await options.loadSdkSummary(null);
      if (platformComponent == null) {
        throw "platformComponent is null";
      }
      hadVerifyError = false;
    }

    if (createNewCompiler) {
      platformComponent!.adoptChildren();
      _incrementalCompiler = new IncrementalCompiler.fromComponent(
        context,
        platformComponent,
      );
    }

    ProcessedOptions incrementalCompilerOptions =
        _incrementalCompiler!.context.options;
    if (!identical(incrementalCompilerOptions.inputs, options.inputs)) {
      // Invalidating the packages uri causes it to recalculate which packages
      // file to use which is what we want.
      _incrementalCompiler!.invalidate(incrementalCompilerOptions.packagesUri);
      incrementalCompilerOptions.inputs.clear();
      incrementalCompilerOptions.inputs.addAll(options.inputs);
    }

    _originalOnDiagnostic =
        options.rawOptionsForTesting.onDiagnostic ??
        options.defaultDiagnosticMessageHandler;
    incrementalCompilerOptions.rawOptionsForTesting.onDiagnostic =
        _onDiagnostic;

    assert(
      options.omitPlatform,
      "Platform must be omitted for the batch compiler.",
    );
    assert(
      !options.hasAdditionalDillModules,
      "Additional dill modules are not supported for the batch compiler.",
    );
    IncrementalCompilerResult compilerResult = await _incrementalCompiler!
        .computeDelta(fullComponent: true);
    await _emitComponent(
      options,
      compilerResult.component,
      message: "Wrote component to ",
    );
    for (CfeDiagnosticMessage error in _errors) {
      if (error.codeName == diag.internalProblemVerificationError.name) {
        hadVerifyError = true;
      }
    }
    return _errors.isEmpty;
  }
}

Future<void> incrementalEntryPoint(List<String> arguments) async {
  installAdditionalTargets();
  await withGlobalOptions("incremental", arguments, true, (
    CompilerContext c,
    _,
  ) {
    // TODO(ahe): Extend this entry point so it can replace
    // batchEntryPoint.
    new IncrementalCompiler(c);
    return Future<void>.value();
  });
}

Future<void> outline(List<String> arguments, {Benchmarker? benchmarker}) async {
  return await runProtectedFromAbort<void>(() async {
    return await withGlobalOptions("outline", arguments, true, (
      CompilerContext c,
      _,
    ) async {
      if (c.options.verbose) {
        print("Building outlines for ${arguments.join(' ')}");
      }
      CompilerResult compilerResult = await generateKernelInternal(
        c,
        buildSummary: true,
        serializeIfBuildingSummary: false,
        buildComponent: false,
        benchmarker: benchmarker,
      );
      Component component = compilerResult.component!;
      await _emitComponent(
        c.options,
        component,
        benchmarker: benchmarker,
        message: "Wrote outline to ",
      );
    });
  });
}

Future<Uri> compile(
  List<String> arguments, {
  Benchmarker? benchmarker,
  bool useIncrementalCompiler = false,
}) async {
  return await runProtectedFromAbort<Uri>(() async {
    return await withGlobalOptions("compile", arguments, true, (
      CompilerContext c,
      _,
    ) async {
      if (c.options.verbose) {
        print("Compiling directly to Kernel: ${arguments.join(' ')}");
      }
      Component component;
      if (useIncrementalCompiler) {
        IncrementalCompiler incrementalCompiler = new IncrementalCompiler(c);
        IncrementalCompilerResult compilerResult = await incrementalCompiler
            .computeDelta();
        component = compilerResult.component;
      } else {
        CompilerResult compilerResult = await generateKernelInternal(
          c,
          benchmarker: benchmarker,
        );
        component = compilerResult.component!;
      }
      Uri uri = await _emitComponent(
        c.options,
        component,
        benchmarker: benchmarker,
        message: "Wrote component to ",
      );
      _benchmarkAstVisitor(component, benchmarker);
      return uri;
    });
  });
}

Future<Uri?> deps(List<String> arguments) async {
  return await runProtectedFromAbort<Uri?>(() async {
    FileSystemDependencyTracker tracker = new FileSystemDependencyTracker();
    return await withGlobalOptions("deps", arguments, true, tracker: tracker, (
      CompilerContext c,
      _,
    ) async {
      if (c.options.verbose) {
        print("Computing deps: ${arguments.join(' ')}");
      }
      await generateKernelInternal(
        c,
        onlyDirectives: true,
        buildComponent: false,
        buildSummary: false,
      );
      return await _emitDeps(tracker, c.options.output);
    });
  });
}

/// Writes the [component] to the URI specified in the compiler options.
Future<Uri> _emitComponent(
  ProcessedOptions options,
  Component component, {
  Benchmarker? benchmarker,
  required String message,
}) async {
  Uri uri = options.output!;
  if (options.omitPlatform) {
    benchmarker?.enterPhase(BenchmarkPhases.omitPlatform);
    component.computeCanonicalNames();
    Component userCode = new Component(
      nameRoot: component.root,
      uriToSource: new Map<Uri, Source>.from(component.uriToSource),
    );
    userCode.setMainMethodAndMode(component.mainMethodName, true);
    for (Library library in component.libraries) {
      if (!library.importUri.isScheme("dart")) {
        userCode.libraries.add(library);
      }
    }
    component = userCode;
  }
  if (uri.isScheme("file")) {
    benchmarker?.enterPhase(BenchmarkPhases.writeComponent);
    await writeComponentToFile(component, uri);
    options.ticker.logMs("${message}${uri.toFilePath()}");
  }
  return uri;
}

Future<Uri?> _emitDeps(
  FileSystemDependencyTracker tracker, [
  Uri? output,
]) async {
  Uri? dFile;
  if (output != null) {
    dFile = new File(new File.fromUri(output).path + ".d").uri;
    await writeDepsFile(output, dFile, tracker.dependencies);
  }
  return dFile;
}

/// Runs a visitor on [component] for benchmarking.
void _benchmarkAstVisitor(Component component, Benchmarker? benchmarker) {
  if (benchmarker != null) {
    // When benchmarking also do a recursive visit of the produced component
    // that does nothing other than visiting everything. Do this to produce
    // a reference point for comparing inference time and serialization time.
    benchmarker.enterPhase(BenchmarkPhases.benchmarkAstVisit);
    component.accept(new EmptyRecursiveVisitorForBenchmarking());
  }
  benchmarker?.enterPhase(BenchmarkPhases.unknown);
}

class EmptyRecursiveVisitorForBenchmarking extends RecursiveVisitor {}

Future<void> compilePlatform(List<String> arguments) async {
  FileSystemDependencyTracker tracker = new FileSystemDependencyTracker();
  await withGlobalOptions(
    "compile_platform",
    arguments,
    false,
    tracker: tracker,
    (CompilerContext c, List<String> restArguments) {
      c.compilingPlatform = true;
      Uri hostPlatform = Uri.base.resolveUri(new Uri.file(restArguments[2]));
      Uri outlineOutput = Uri.base.resolveUri(new Uri.file(restArguments[4]));
      return compilePlatformInternal(
        c,
        tracker,
        c.options.output!,
        outlineOutput,
        hostPlatform,
      );
    },
  );
}

Future<void> compilePlatformInternal(
  CompilerContext c,
  FileSystemDependencyTracker tracker,
  Uri fullOutput,
  Uri outlineOutput,
  Uri hostPlatform,
) async {
  if (c.options.verbose) {
    print("Generating outline of ${c.options.sdkRoot} into $outlineOutput");
    print("Compiling ${c.options.sdkRoot} to $fullOutput");
  }

  CompilerResult result = await generateKernelInternal(
    c,
    buildSummary: true,
    buildComponent: true,
  );
  new File.fromUri(outlineOutput).writeAsBytesSync(result.summary!);
  c.options.ticker.logMs("Wrote outline to ${outlineOutput.toFilePath()}");

  verifyComponent(
    c.options.target,
    VerificationStage.afterModularTransformations,
    result.component!,
  );
  await writeComponentToFile(result.component!, fullOutput);

  c.options.ticker.logMs("Wrote component to ${fullOutput.toFilePath()}");

  if (c.options.emitDeps) {
    List<Uri> deps = tracker.dependencies.toList();
    for (Uri dependency in await computeHostDependencies(hostPlatform)) {
      // Add the dependencies of the compiler's own sources.
      if (dependency != outlineOutput) {
        // We're computing the dependencies for [outlineOutput], so we shouldn't
        // include it in the deps file.
        deps.add(dependency);
      }
    }
    await writeDepsFile(
      fullOutput,
      new File(new File.fromUri(fullOutput).path + ".d").uri,
      deps,
    );
  }
}

Future<List<Uri>> computeHostDependencies(Uri hostPlatform) {
  // Do not try to parse compile_platform if it was precompiled into a binary.
  if (!Platform.script.toFilePath().endsWith('.dart')) {
    return Future.value([]);
  }

  // Returns a list of source files that make up the Fasta compiler (the files
  // the Dart VM reads to run Fasta). Until Fasta is self-hosting (in strong
  // mode), this is only an approximation, albeit accurate.  Once Fasta is
  // self-hosting, this isn't an approximation. Regardless, strong mode
  // shouldn't affect which files are read.
  Target? hostTarget = getTarget("vm", new TargetFlags());
  return getDependencies(
    Platform.script,
    platform: hostPlatform,
    target: hostTarget,
  );
}

Future<void> writeDepsFile(
  Uri output,
  Uri depsFile,
  List<Uri> allDependencies,
) async {
  if (allDependencies.isEmpty) return;
  String toRelativeFilePath(Uri uri) {
    // Ninja expects to find file names relative to the current working
    // directory. We've tried making them relative to the deps file, but that
    // doesn't work for downstream projects. Making them absolute also
    // doesn't work.
    //
    // We can test if it works by running ninja twice, for example:
    //
    //     ninja -C xcodebuild/ReleaseX64 -d explain compile_platform
    //     ninja -C xcodebuild/ReleaseX64 -d explain compile_platform
    //
    // The second time, ninja should say:
    //
    //     ninja: Entering directory `xcodebuild/ReleaseX64'
    //     ninja: no work to do.
    //
    // It's broken if it says something like this:
    //
    //     ninja explain: expected depfile 'vm_platform.dill.d' to mention \
    //     'vm_platform.dill', got '/.../xcodebuild/ReleaseX64/vm_platform.dill'
    return Uri.parse(relativizeUri(Uri.base, uri, isWindows)).toFilePath();
  }

  StringBuffer sb = new StringBuffer();
  sb.write(toRelativeFilePath(output));
  sb.write(":");
  List<String> paths = new List<String>.generate(
    allDependencies.length,
    (int i) => toRelativeFilePath(allDependencies[i]),
    growable: false,
  );
  // Sort the relative paths to ease analyzing future changes to this code.
  paths.sort();
  String? previous;
  for (String path in paths) {
    // Check for and omit duplicates.
    if (path != previous) {
      previous = path;
      sb.write(" \\\n  ");
      sb.write(path);
    }
  }
  sb.writeln();
  await new File.fromUri(depsFile).writeAsString("$sb");
}