pkg/dev_compiler/test/hot_reload_suite.dart

// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'dart:math';

import 'package:_fe_analyzer_shared/src/util/relativize.dart' as fe_shared;
import 'package:args/args.dart';
import 'package:dev_compiler/dev_compiler.dart' as ddc_names
    show libraryUriToJsIdentifier;
import 'package:front_end/src/compute_platform_binaries_location.dart' as fe;
import 'package:reload_test/ddc_helpers.dart' as ddc_helpers;
import 'package:reload_test/frontend_server_controller.dart';
import 'package:reload_test/hot_reload_memory_filesystem.dart';
import 'package:reload_test/test_helpers.dart';

// Set an arbitrary cap on generations.
final globalMaxGenerations = 100;

final testTimeoutSeconds = 10;

final argParser = ArgParser()
  ..addOption('runtime',
      abbr: 'r',
      defaultsTo: 'd8',
      allowed: RuntimePlatforms.values.map((v) => v.text),
      help: 'runtime platform used to run tests.')
  ..addOption('named-configuration',
      abbr: 'n',
      defaultsTo: 'no-configuration',
      help: 'configuration name to use for emitting test result files.')
  ..addOption('output-directory', help: 'directory to emit test results files.')
  ..addFlag('debug',
      abbr: 'd',
      defaultsTo: false,
      negatable: true,
      help: 'enables additional debug behavior and logging.')
  ..addFlag('verbose',
      abbr: 'v',
      defaultsTo: true,
      negatable: true,
      help: 'enables verbose logging.');

late final bool verbose;
late final bool debug;

/// TODO(markzipan): Add arg parsing for additional execution modes
/// (chrome, VM) and diffs across generations.
Future<void> main(List<String> args) async {
  final argResults = argParser.parse(args);
  final runtimePlatform =
      RuntimePlatforms.values.byName(argResults['runtime'] as String);
  debug = argResults['debug'] as bool;
  verbose = argResults['verbose'] as bool;

  // Used to communicate individual test failures to our test bots.
  final emitTestResultsJson = argResults['output-directory'] != null;
  final buildRootUri = fe.computePlatformBinariesLocation(forceBuildDir: true);
  // We can use the outline instead of the full SDK dill here.
  final ddcPlatformDillUri = buildRootUri.resolve('ddc_outline.dill');
  final vmPlatformDillUri = buildRootUri.resolve('vm_platform_strong.dill');

  final sdkRoot = Platform.script.resolve('../../../');
  final packageConfigUri = sdkRoot.resolve('.dart_tool/package_config.json');
  final allTestsUri = sdkRoot.resolve('tests/hot_reload/');
  final soundStableDartSdkJsUri =
      buildRootUri.resolve('gen/utils/ddc/stable/sdk/ddc/dart_sdk.js');
  final ddcModuleLoaderJsUri =
      sdkRoot.resolve('pkg/dev_compiler/lib/js/ddc/ddc_module_loader.js');
  final d8Config = ddc_helpers.D8Configuration(sdkRoot);

  // Contains generated code for all tests.
  final generatedCodeDir = Directory.systemTemp.createTempSync();
  final generatedCodeUri = generatedCodeDir.uri;
  _debugPrint('See generated hot reload framework code in $generatedCodeUri');

  // The snapshot directory is a staging area the test framework uses to
  // construct a compile-able test app across reload/restart generations.
  final snapshotDir = Directory.fromUri(generatedCodeUri.resolve('.snapshot/'));
  snapshotDir.createSync();
  final snapshotUri = snapshotDir.uri;

  // Contains files emitted from Frontend Server compiles and recompiles.
  final frontendServerEmittedFilesDirUri = generatedCodeUri.resolve('.fes/');
  Directory.fromUri(frontendServerEmittedFilesDirUri).createSync();
  final outputDillUri = frontendServerEmittedFilesDirUri.resolve('output.dill');
  final outputIncrementalDillUri =
      frontendServerEmittedFilesDirUri.resolve('output_incremental.dill');

  // TODO(markzipan): Support custom entrypoints.
  final snapshotEntrypointUri = snapshotUri.resolve('main.dart');
  final filesystemRootUri = snapshotUri;
  final filesystemScheme = 'hot-reload-test';
  final snapshotEntrypointLibraryName = fe_shared.relativizeUri(
      filesystemRootUri, snapshotEntrypointUri, fe_shared.isWindows);
  final snapshotEntrypointWithScheme =
      '$filesystemScheme:///$snapshotEntrypointLibraryName';

  _print('Initializing the Frontend Server.');
  HotReloadFrontendServerController controller;
  final commonArgs = [
    '--incremental',
    '--filesystem-root=${snapshotUri.toFilePath()}',
    '--filesystem-scheme=$filesystemScheme',
    '--output-dill=${outputDillUri.toFilePath()}',
    '--output-incremental-dill=${outputIncrementalDillUri.toFilePath()}',
    '--packages=${packageConfigUri.toFilePath()}',
    '--sdk-root=${sdkRoot.toFilePath()}',
    '--verbosity=${verbose ? 'all' : 'info'}',
  ];
  switch (runtimePlatform) {
    case RuntimePlatforms.d8:
      final ddcPlatformDillFromSdkRoot = fe_shared.relativizeUri(
          sdkRoot, ddcPlatformDillUri, fe_shared.isWindows);
      final fesArgs = [
        ...commonArgs,
        '--dartdevc-module-format=ddc',
        '--platform=$ddcPlatformDillFromSdkRoot',
        '--target=dartdevc',
      ];
      controller = HotReloadFrontendServerController(fesArgs);
      break;
    case RuntimePlatforms.chrome:
      throw Exception('Unsupported platform: $runtimePlatform');
    case RuntimePlatforms.vm:
      final vmPlatformDillFromSdkRoot = fe_shared.relativizeUri(
          sdkRoot, vmPlatformDillUri, fe_shared.isWindows);
      final fesArgs = [
        ...commonArgs,
        '--platform=$vmPlatformDillFromSdkRoot',
        '--target=vm',
      ];
      controller = HotReloadFrontendServerController(fesArgs);
      break;
  }
  controller.start();

  Future<void> shutdown() async {
    // Persist the temp directory for debugging.
    await controller.stop();
    _print('Frontend Server has shut down.');
    if (!debug) {
      generatedCodeDir.deleteSync(recursive: true);
    }
  }

  final testOutcomes = <TestResultOutcome>[];
  final validTestSourceName = RegExp(r'.*[a-zA-Z0-9]+.[0-9]+.dart');
  for (var testDir in Directory.fromUri(allTestsUri).listSync()) {
    if (testDir is! Directory) {
      if (testDir is File) {
        // Ignore Dart source files, which may be imported as helpers
        continue;
      }
      throw Exception(
          'Non-directory or file entity found in ${allTestsUri.toFilePath()}: $testDir');
    }
    final testDirParts = testDir.uri.pathSegments;
    final testName = testDirParts[testDirParts.length - 2];

    var outcome = TestResultOutcome(
      configuration: argResults['named-configuration'] as String,
      testName: testName,
    );
    var stopwatch = Stopwatch()..start();

    final tempUri = generatedCodeUri.resolve('$testName/');
    Directory.fromUri(tempUri).createSync();

    _print('Generating test assets.', label: testName);
    _debugPrint('Emitting JS code to ${tempUri.toFilePath()}.',
        label: testName);

    var filesystem = HotReloadMemoryFilesystem(tempUri);

    // Count the number of generations for this test.
    //
    // Assumes all files are named like '$name.$integer.dart', where 0 is the
    // first generation.
    //
    // TODO(markzipan): Account for subdirectories.
    var maxGenerations = 0;
    var testConfig = ReloadTestConfiguration();
    for (var file in testDir.listSync()) {
      if (file is File) {
        final fileName = file.uri.pathSegments.last;
        // Process config files.
        if (fileName == 'config.json') {
          testConfig = ReloadTestConfiguration.fromJsonFile(file.uri);
        } else if (fileName.endsWith('.dart')) {
          if (!validTestSourceName.hasMatch(fileName)) {
            throw Exception('Invalid test source file name: $fileName\n'
                'Valid names look like "file_name.10.dart".');
          }
          var strippedName =
              file.path.substring(0, file.path.length - '.dart'.length);
          var parts = strippedName.split('.');
          var generationId = int.parse(parts[parts.length - 1]);
          maxGenerations = max(maxGenerations, generationId);
        }
      }
    }
    if (maxGenerations > globalMaxGenerations) {
      throw Exception('Too many generations specified in test '
          '(requested: $maxGenerations, max: $globalMaxGenerations).');
    }

    // Skip this test directory if this platform is excluded.
    if (testConfig.excludedPlaforms.contains(runtimePlatform)) {
      _print('Skipping test on platform: ${runtimePlatform.text}',
          label: testName);
      continue;
    }

    // TODO(markzipan): replace this with a test-configurable main entrypoint.
    final mainDartFilePath = testDir.uri.resolve('main.dart').toFilePath();
    _debugPrint('Test entrypoint: $mainDartFilePath', label: testName);
    _print('Generating code over ${maxGenerations + 1} generations.',
        label: testName);

    // Generate hot reload/restart generations as subdirectories in a loop.
    var currentGeneration = 0;
    while (currentGeneration <= maxGenerations) {
      _debugPrint('Entering generation $currentGeneration', label: testName);
      var updatedFilesInCurrentGeneration = <String>[];

      // Copy all files in this generation to the snapshot directory with their
      // names restored (e.g., path/to/main' from 'path/to/main.0.dart).
      // TODO(markzipan): support subdirectories.
      _debugPrint(
          'Copying Dart files to snapshot directory: ${snapshotUri.toFilePath()}',
          label: testName);
      for (var file in testDir.listSync()) {
        // Convert a name like `/path/foo.bar.25.dart` to `/path/foo.bar.dart`.
        if (file is File && file.path.endsWith('.dart')) {
          final baseName = file.uri.pathSegments.last;
          final parts = baseName.split('.');
          final generationId = int.parse(parts[parts.length - 2]);
          if (generationId == currentGeneration) {
            // Reconstruct the name of the file without generation indicators.
            parts.removeLast(); // Remove `.dart`.
            parts.removeLast(); // Remove the generation id.
            parts.add('.dart'); // Re-add `.dart`.
            final restoredName = parts.join();
            final fileSnapshotUri = snapshotUri.resolve(restoredName);
            final relativeSnapshotPath = fe_shared.relativizeUri(
                filesystemRootUri, fileSnapshotUri, fe_shared.isWindows);
            final snapshotPathWithScheme =
                '$filesystemScheme:///$relativeSnapshotPath';
            updatedFilesInCurrentGeneration.add(snapshotPathWithScheme);
            file.copySync(fileSnapshotUri.toFilePath());
          }
        }
      }
      _print(
          'Updated files in generation $currentGeneration: '
          '[${updatedFilesInCurrentGeneration.join(', ')}]',
          label: testName);

      // The first generation calls `compile`, but subsequent ones call
      // `recompile`.
      // Likewise, use the incremental output directory for `recompile` calls.
      String outputDirectoryPath;
      _print(
          'Compiling generation $currentGeneration with the Frontend Server.',
          label: testName);
      if (currentGeneration == 0) {
        _debugPrint(
            'Compiling snapshot entrypoint: $snapshotEntrypointWithScheme',
            label: testName);
        outputDirectoryPath = outputDillUri.toFilePath();
        await controller.sendCompileAndAccept(snapshotEntrypointWithScheme);
      } else {
        _debugPrint(
            'Recompiling snapshot entrypoint: $snapshotEntrypointWithScheme',
            label: testName);
        outputDirectoryPath = outputIncrementalDillUri.toFilePath();
        // TODO(markzipan): Add logic to reject bad compiles.
        await controller.sendRecompileAndAccept(snapshotEntrypointWithScheme,
            invalidatedFiles: updatedFilesInCurrentGeneration);
      }
      _debugPrint(
          'Frontend Server successfully compiled outputs to: '
          '$outputDirectoryPath',
          label: testName);

      if (runtimePlatform.emitsJS) {
        // Update the memory filesystem with the newly-created JS files
        _print(
            'Loading generation $currentGeneration files '
            'into the memory filesystem.',
            label: testName);
        final codeFile = File('$outputDirectoryPath.sources');
        final manifestFile = File('$outputDirectoryPath.json');
        final sourcemapFile = File('$outputDirectoryPath.map');
        filesystem.update(
          codeFile,
          manifestFile,
          sourcemapFile,
          generation: '$currentGeneration',
        );

        // Write JS files and sourcemaps to their respective generation.
        _print('Writing generation $currentGeneration assets.',
            label: testName);
        _debugPrint('Writing JS assets to ${tempUri.toFilePath()}',
            label: testName);
        filesystem.writeToDisk(tempUri, generation: '$currentGeneration');
      } else {
        final dillOutputDir =
            Directory.fromUri(tempUri.resolve('generation$currentGeneration'));
        dillOutputDir.createSync();
        final dillOutputUri = dillOutputDir.uri.resolve('$testName.dill');
        File(outputDirectoryPath).copySync(dillOutputUri.toFilePath());
        // Write dills their respective generation.
        _print('Writing generation $currentGeneration assets.',
            label: testName);
        _debugPrint('Writing dill to ${dillOutputUri.toFilePath()}',
            label: testName);
      }
      currentGeneration++;
    }

    _print('Finished emitting assets.', label: testName);

    final testOutputStreamController = StreamController<List<int>>();
    final testOutputBuffer = StringBuffer();
    testOutputStreamController.stream
        .transform(utf8.decoder)
        .listen(testOutputBuffer.write);
    var testPassed = false;
    switch (runtimePlatform) {
      case RuntimePlatforms.d8:
        // Run the compiled JS generations with D8.
        // TODO(markzipan): Add logic for evaluating with Chrome or the VM.
        _print('Creating D8 hot reload test suite.', label: testName);

        final d8Suite = D8SuiteRunner(
          config: d8Config,
          bootstrapJsUri: tempUri.resolve('generation0/bootstrap.js'),
          entrypointLibraryExportName:
              ddc_names.libraryUriToJsIdentifier(snapshotEntrypointUri),
          dartSdkJsUri: soundStableDartSdkJsUri,
          ddcModuleLoaderJsUri: ddcModuleLoaderJsUri,
          outputSink: IOSink(testOutputStreamController.sink),
        );
        await d8Suite.setupTest(
          testName: testName,
          scriptDescriptors: filesystem.scriptDescriptorForBootstrap,
          generationToModifiedFiles: filesystem.generationsToModifiedFilePaths,
        );
        final d8ExitCode = await d8Suite.runTest(testName: testName);
        testPassed = d8ExitCode == 0;
        await d8Suite.teardownTest(testName: testName);
        break;
      case RuntimePlatforms.chrome:
      case RuntimePlatforms.vm:
        final firstGenerationDillUri =
            tempUri.resolve('generation0/$testName.dill');
        // Start the VM at generation 0.
        final vmArgs = [
          '--enable-vm-service=0', // 0 avoids port collisions.
          '--disable-service-auth-codes',
          '--disable-dart-dev',
          firstGenerationDillUri.toFilePath(),
        ];
        final vm = await Process.start(Platform.executable, vmArgs);
        _debugPrint(
            'Starting VM with command: ${Platform.executable} ${vmArgs.join(" ")}',
            label: testName);
        vm.stdout
            .transform(utf8.decoder)
            .transform(LineSplitter())
            .listen((String line) {
          _debugPrint('VM stdout: $line', label: testName);
          testOutputBuffer.writeln(line);
        });
        vm.stderr
            .transform(utf8.decoder)
            .transform(LineSplitter())
            .listen((String err) {
          _debugPrint('VM stderr: $err', label: testName);
          testOutputBuffer.writeln(err);
        });
        _print('Executing VM test.', label: testName);
        final vmExitCode = await vm.exitCode
            .timeout(Duration(seconds: testTimeoutSeconds), onTimeout: () {
          final timeoutText =
              'Test timed out after $testTimeoutSeconds seconds.';
          _print(timeoutText, label: testName);
          testOutputBuffer.writeln(timeoutText);
          vm.kill();
          return 1;
        });
        testPassed = vmExitCode == 0;
    }

    stopwatch.stop();
    final testOutput = testOutputBuffer.toString();
    outcome.elapsedTime = stopwatch.elapsed;
    outcome.matchedExpectations = testPassed;
    outcome.testOutput = testOutput;
    testOutcomes.add(outcome);
    if (testPassed) {
      _print('PASSED with:\n$testOutput', label: testName);
    } else {
      _print('FAILED with:\n$testOutput', label: testName);
      await shutdown();
      exit(1);
    }
  }

  await shutdown();
  _print('Testing complete.');

  if (emitTestResultsJson) {
    final testOutcomeResults = testOutcomes.map((o) => o.toRecordJson());
    final testOutcomeLogs = testOutcomes.map((o) => o.toLogJson());
    final testResultsOutputDir =
        Uri.directory(argResults['output-directory'] as String);
    _print('Saving test results to ${testResultsOutputDir.toFilePath()}.');

    // Test outputs must have one JSON blob per line and be newline-terminated.
    final testResultsUri = testResultsOutputDir.resolve('results.json');
    final testResultsSink = File.fromUri(testResultsUri).openWrite();
    testOutcomeResults.forEach(testResultsSink.writeln);
    await testResultsSink.flush();
    await testResultsSink.close();

    final testLogsUri = testResultsOutputDir.resolve('logs.json');
    if (Platform.isWindows) {
      // TODO(55297): Logs are disabled on windows until this but is fixed.
      _print('Logs are not written on Windows. '
          'See: https://github.com/dart-lang/sdk/issues/55297');
    } else {
      final testLogsSink = File.fromUri(testLogsUri).openWrite();
      testOutcomeLogs.forEach(testLogsSink.writeln);
      await testLogsSink.flush();
      await testLogsSink.close();
    }
    _print('Emitted logs to ${testResultsUri.toFilePath()} '
        'and ${testLogsUri.toFilePath()}.');
  }
}

/// Runs the [command] with [args] in [environment].
///
/// Will echo the commands to the console before running them when running in
/// `verbose` mode.
Future<Process> startProcess(String name, String command, List<String> args,
    {Map<String, String> environment = const {},
    ProcessStartMode mode = ProcessStartMode.normal}) {
  if (verbose) {
    print('Running $name:\n$command ${args.join(' ')}\n');
    if (environment.isNotEmpty) {
      var environmentVariables =
          environment.entries.map((e) => '${e.key}: ${e.value}').join('\n');
      print('With environment:\n$environmentVariables\n');
    }
  }
  return Process.start(command, args, mode: mode, environment: environment);
}

/// Prints messages if 'verbose' mode is enabled.
void _print(String message, {String? label}) {
  if (verbose) {
    final labelText = label == null ? '' : '($label)';
    print('hot_reload_test$labelText: $message');
  }
}

/// Prints messages if 'debug' mode is enabled.
void _debugPrint(String message, {String? label}) {
  if (debug) {
    final labelText = label == null ? '' : '($label)';
    print('DEBUG$labelText: $message');
  }
}

abstract class HotReloadSuiteRunner {
  final String entrypointModuleName;
  final String entrypointLibraryExportName;
  final Uri dartSdkJsUri;
  final Uri ddcModuleLoaderJsUri;
  final StreamSink<List<int>> outputSink;

  HotReloadSuiteRunner({
    required this.entrypointModuleName,
    required this.entrypointLibraryExportName,
    required this.dartSdkJsUri,
    required this.ddcModuleLoaderJsUri,
    required this.outputSink,
  });

  /// Logic that needs to be run before every test begins.
  ///
  /// [scriptDescriptors] and [generationToModifiedFiles] are only used for
  /// DDC-based execution environments.
  Future<void> setupTest(
      {String? testName,
      List<Map<String, String?>>? scriptDescriptors,
      Map<String, List<String>>? generationToModifiedFiles});

  /// Executes a test.
  Future<int> runTest({String? testName});

  /// Logic that needs to be run after every test completes.
  Future<void> teardownTest({String? testName});
}

class D8SuiteRunner implements HotReloadSuiteRunner {
  final ddc_helpers.D8Configuration config;
  final Uri bootstrapJsUri;
  @override
  final String entrypointModuleName;
  @override
  final String entrypointLibraryExportName;
  @override
  final Uri dartSdkJsUri;
  @override
  final Uri ddcModuleLoaderJsUri;
  @override
  final StreamSink<List<int>> outputSink;

  D8SuiteRunner._({
    required this.config,
    required this.bootstrapJsUri,
    required this.entrypointModuleName,
    required this.entrypointLibraryExportName,
    required this.dartSdkJsUri,
    required this.ddcModuleLoaderJsUri,
    required this.outputSink,
  });

  factory D8SuiteRunner({
    required ddc_helpers.D8Configuration config,
    required Uri bootstrapJsUri,
    String entrypointModuleName = 'main.dart',
    String entrypointLibraryExportName = 'main',
    required Uri dartSdkJsUri,
    required Uri ddcModuleLoaderJsUri,
    StreamSink<List<int>>? outputSink,
  }) {
    return D8SuiteRunner._(
      config: config,
      entrypointModuleName: entrypointModuleName,
      entrypointLibraryExportName: entrypointLibraryExportName,
      bootstrapJsUri: bootstrapJsUri,
      dartSdkJsUri: dartSdkJsUri,
      ddcModuleLoaderJsUri: ddcModuleLoaderJsUri,
      outputSink: outputSink ?? stdout,
    );
  }

  String _generateBootstrapper({
    required List<Map<String, String?>> scriptDescriptors,
    required Map<String, List<String>> generationToModifiedFiles,
  }) {
    return ddc_helpers.generateD8Bootstrapper(
      ddcModuleLoaderJsPath: escapedString(ddcModuleLoaderJsUri.toFilePath()),
      dartSdkJsPath: escapedString(dartSdkJsUri.toFilePath()),
      entrypointModuleName: escapedString(entrypointModuleName),
      entrypointLibraryExportName: escapedString(entrypointLibraryExportName),
      scriptDescriptors: scriptDescriptors,
      modifiedFilesPerGeneration: generationToModifiedFiles,
    );
  }

  @override
  Future<void> setupTest({
    String? testName,
    List<Map<String, String?>>? scriptDescriptors,
    Map<String, List<String>>? generationToModifiedFiles,
  }) async {
    _print('Preparing to run D8 test.', label: testName);
    if (scriptDescriptors == null || generationToModifiedFiles == null) {
      throw ArgumentError('D8SuiteRunner requires that "scriptDescriptors" '
          'and "generationToModifiedFiles" be provided during setup.');
    }
    final d8BootstrapJS = _generateBootstrapper(
        scriptDescriptors: scriptDescriptors,
        generationToModifiedFiles: generationToModifiedFiles);
    File.fromUri(bootstrapJsUri).writeAsStringSync(d8BootstrapJS);
    _debugPrint('Writing D8 bootstrapper: $bootstrapJsUri', label: testName);
  }

  @override
  Future<int> runTest({String? testName}) async {
    final process = await startProcess('D8', config.binary.toFilePath(), [
      config.sealNativeObjectScript.toFilePath(),
      config.preamblesScript.toFilePath(),
      bootstrapJsUri.toFilePath()
    ]);
    unawaited(process.stdout.pipe(outputSink));
    return process.exitCode;
  }

  @override
  Future<void> teardownTest({String? testName}) async {}
}