pkg/analysis_server/lib/src/context_manager.dart

// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:collection';
import 'dart:core';

import 'package:analysis_server/src/services/correction/fix/data_driven/transform_set_parser.dart';
import 'package:analyzer/error/listener.dart';
import 'package:analyzer/file_system/file_system.dart';
import 'package:analyzer/instrumentation/instrumentation.dart';
import 'package:analyzer/src/dart/analysis/analysis_context_collection.dart';
import 'package:analyzer/src/dart/analysis/byte_store.dart';
import 'package:analyzer/src/dart/analysis/driver.dart';
import 'package:analyzer/src/dart/analysis/driver_based_analysis_context.dart';
import 'package:analyzer/src/dart/analysis/performance_logger.dart';
import 'package:analyzer/src/generated/java_engine.dart';
import 'package:analyzer/src/generated/sdk.dart';
import 'package:analyzer/src/generated/source.dart';
import 'package:analyzer/src/generated/source_io.dart';
import 'package:analyzer/src/lint/linter.dart';
import 'package:analyzer/src/lint/pub.dart';
import 'package:analyzer/src/manifest/manifest_validator.dart';
import 'package:analyzer/src/pubspec/pubspec_validator.dart';
import 'package:analyzer/src/task/options.dart';
import 'package:analyzer/src/util/file_paths.dart' as file_paths;
import 'package:analyzer/src/workspace/bazel.dart';
import 'package:analyzer/src/workspace/bazel_watcher.dart';
import 'package:analyzer_plugin/protocol/protocol_common.dart' as protocol;
import 'package:analyzer_plugin/utilities/analyzer_converter.dart';
import 'package:path/path.dart' as path;
import 'package:watcher/watcher.dart';
import 'package:yaml/yaml.dart';

/// Enables watching of files generated by Bazel.
///
/// TODO(michalt): This is a temporary flag that we use to disable this
/// functionality due its performance issues. We plan to benchmark and optimize
/// it and re-enable it everywhere.
/// Not private to enable testing.
/// NB: If you set this to `false` remember to disable the
/// `test/integration/serve/bazel_changes_test.dart`.
var experimentalEnableBazelWatching = true;

/// Class that maintains a mapping from included/excluded paths to a set of
/// folders that should correspond to analysis contexts.
abstract class ContextManager {
  /// Get the callback interface used to create, destroy, and update contexts.
  ContextManagerCallbacks get callbacks;

  /// Set the callback interface used to create, destroy, and update contexts.
  set callbacks(ContextManagerCallbacks value);

  /// A table mapping [Folder]s to the [AnalysisDriver]s associated with them.
  Map<Folder, AnalysisDriver> get driverMap;

  /// Return the list of excluded paths (folders and files) most recently passed
  /// to [setRoots].
  List<String> get excludedPaths;

  /// Return the list of included paths (folders and files) most recently passed
  /// to [setRoots].
  List<String> get includedPaths;

  /// Return the existing analysis context that should be used to analyze the
  /// given [path], or `null` if the [path] is not analyzed in any of the
  /// created analysis contexts.
  DriverBasedAnalysisContext? getContextFor(String path);

  /// Return the [AnalysisDriver] for the "innermost" context whose associated
  /// folder is or contains the given path.  ("innermost" refers to the nesting
  /// of contexts, so if there is a context for path /foo and a context for
  /// path /foo/bar, then the innermost context containing /foo/bar/baz.dart is
  /// the context for /foo/bar.)
  ///
  /// If no driver contains the given path, `null` is returned.
  AnalysisDriver? getDriverFor(String path);

  /// Return `true` if the file or directory with the given [path] will be
  /// analyzed in one of the analysis contexts.
  bool isAnalyzed(String path);

  /// Rebuild the set of contexts from scratch based on the data last sent to
  /// [setRoots].
  void refresh();

  /// Change the set of paths which should be used as starting points to
  /// determine the context directories.
  void setRoots(List<String> includedPaths, List<String> excludedPaths);
}

/// Callback interface used by [ContextManager] to (a) request that contexts be
/// created, destroyed or updated, (b) inform the client when "pub list"
/// operations are in progress, and (c) determine which files should be
/// analyzed.
///
/// TODO(paulberry): eliminate this interface, and instead have [ContextManager]
/// operations return data structures describing how context state should be
/// modified.
abstract class ContextManagerCallbacks {
  /// Called after analysis contexts are created, usually when new analysis
  /// roots are set, or after detecting a change that required rebuilding
  /// the set of analysis contexts.
  void afterContextsCreated();

  /// Called after analysis contexts are destroyed.
  void afterContextsDestroyed();

  /// An [event] was processed, so analysis state might be different now.
  void afterWatchEvent(WatchEvent event);

  /// The given [file] was removed.
  void applyFileRemoved(String file);

  /// Sent the given watch [event] to any interested plugins.
  void broadcastWatchEvent(WatchEvent event);

  /// Add listeners to the [driver]. This must be the only listener.
  ///
  /// TODO(scheglov) Just pass results in here?
  void listenAnalysisDriver(AnalysisDriver driver);

  /// Record error information for the file with the given [path].
  void recordAnalysisErrors(String path, List<protocol.AnalysisError> errors);
}

/// Class that maintains a mapping from included/excluded paths to a set of
/// folders that should correspond to analysis contexts.
class ContextManagerImpl implements ContextManager {
  /// The [ResourceProvider] using which paths are converted into [Resource]s.
  final ResourceProvider resourceProvider;

  /// The manager used to access the SDK that should be associated with a
  /// particular context.
  final DartSdkManager sdkManager;

  /// The storage for cached results.
  final ByteStore _byteStore;

  /// The logger used to create analysis contexts.
  final PerformanceLog _performanceLog;

  /// The scheduler used to create analysis contexts, and report status.
  final AnalysisDriverScheduler _scheduler;

  /// The current set of analysis contexts, or `null` if the context roots have
  /// not yet been set.
  AnalysisContextCollectionImpl? _collection;

  /// The context used to work with file system paths.
  path.Context pathContext;

  /// The list of excluded paths (folders and files) most recently passed to
  /// [setRoots].
  @override
  List<String> excludedPaths = <String>[];

  /// The list of included paths (folders and files) most recently passed to
  /// [setRoots].
  @override
  List<String> includedPaths = <String>[];

  /// The instrumentation service used to report instrumentation data.
  final InstrumentationService _instrumentationService;

  @override
  ContextManagerCallbacks callbacks = NoopContextManagerCallbacks();

  @override
  final Map<Folder, AnalysisDriver> driverMap =
      HashMap<Folder, AnalysisDriver>();

  /// Stream subscription we are using to watch each analysis root directory for
  /// changes.
  final Map<Folder, StreamSubscription<WatchEvent>> changeSubscriptions =
      <Folder, StreamSubscription<WatchEvent>>{};

  /// For each folder, stores the subscription to the Bazel workspace so that we
  /// can establish watches for the generated files.
  final bazelSearchSubscriptions =
      <Folder, StreamSubscription<BazelSearchInfo>>{};

  /// The watcher service running in a separate isolate to watch for changes
  /// to files generated by Bazel.
  ///
  /// Might be `null` if watching Bazel files is not enabled.
  BazelFileWatcherService? bazelWatcherService;

  /// The subscription to changes in the files watched by [bazelWatcherService].
  ///
  /// Might be `null` if watching Bazel files is not enabled.
  StreamSubscription<List<WatchEvent>>? bazelWatcherSubscription;

  /// For each [Folder] store which files are being watched. This allows us to
  /// clean up when we destroy a context.
  final bazelWatchedPathsPerFolder = <Folder, _BazelWatchedFiles>{};

  ContextManagerImpl(this.resourceProvider, this.sdkManager, this._byteStore,
      this._performanceLog, this._scheduler, this._instrumentationService,
      {required enableBazelWatcher})
      : pathContext = resourceProvider.pathContext {
    if (enableBazelWatcher) {
      bazelWatcherService = BazelFileWatcherService(_instrumentationService);
      bazelWatcherSubscription = bazelWatcherService!.events
          .listen((events) => _handleBazelWatchEvents(events));
    }
  }

  @override
  DriverBasedAnalysisContext? getContextFor(String path) {
    try {
      return _collection?.contextFor(path);
    } on StateError {
      return null;
    }
  }

  @override
  AnalysisDriver? getDriverFor(String path) {
    return getContextFor(path)?.driver;
  }

  @override
  bool isAnalyzed(String path) {
    var collection = _collection;
    if (collection == null) {
      return false;
    }

    return collection.contexts.any(
      (context) => context.contextRoot.isAnalyzed(path),
    );
  }

  @override
  void refresh() {
    _createAnalysisContexts();
  }

  @override
  void setRoots(List<String> includedPaths, List<String> excludedPaths) {
    if (_rootsAreUnchanged(includedPaths, excludedPaths)) {
      return;
    }

    this.includedPaths = includedPaths;
    this.excludedPaths = excludedPaths;

    _createAnalysisContexts();
  }

  /// Use the given analysis [driver] to analyze the content of the analysis
  /// options file at the given [path].
  void _analyzeAnalysisOptionsYaml(AnalysisDriver driver, String path) {
    var convertedErrors = const <protocol.AnalysisError>[];
    try {
      var content = _readFile(path);
      var lineInfo = _computeLineInfo(content);
      var errors = analyzeAnalysisOptions(
          resourceProvider.getFile(path).createSource(),
          content,
          driver.sourceFactory);
      var converter = AnalyzerConverter();
      convertedErrors = converter.convertAnalysisErrors(errors,
          lineInfo: lineInfo, options: driver.analysisOptions);
    } catch (exception) {
      // If the file cannot be analyzed, fall through to clear any previous
      // errors.
    }
    callbacks.recordAnalysisErrors(path, convertedErrors);
  }

  /// Use the given analysis [driver] to analyze the content of the
  /// AndroidManifest file at the given [path].
  void _analyzeAndroidManifestXml(AnalysisDriver driver, String path) {
    var convertedErrors = const <protocol.AnalysisError>[];
    try {
      var content = _readFile(path);
      var validator =
          ManifestValidator(resourceProvider.getFile(path).createSource());
      var lineInfo = _computeLineInfo(content);
      var errors = validator.validate(
          content, driver.analysisOptions.chromeOsManifestChecks);
      var converter = AnalyzerConverter();
      convertedErrors = converter.convertAnalysisErrors(errors,
          lineInfo: lineInfo, options: driver.analysisOptions);
    } catch (exception) {
      // If the file cannot be analyzed, fall through to clear any previous
      // errors.
    }
    callbacks.recordAnalysisErrors(path, convertedErrors);
  }

  /// Use the given analysis [driver] to analyze the content of the
  /// data file at the given [path].
  void _analyzeFixDataYaml(AnalysisDriver driver, String path) {
    var convertedErrors = const <protocol.AnalysisError>[];
    try {
      var file = resourceProvider.getFile(path);
      var packageName = file.parent2.parent2.shortName;
      var content = _readFile(path);
      var errorListener = RecordingErrorListener();
      var errorReporter = ErrorReporter(errorListener, file.createSource());
      var parser = TransformSetParser(errorReporter, packageName);
      parser.parse(content);
      var converter = AnalyzerConverter();
      convertedErrors = converter.convertAnalysisErrors(errorListener.errors,
          lineInfo: _computeLineInfo(content), options: driver.analysisOptions);
    } catch (exception) {
      // If the file cannot be analyzed, fall through to clear any previous
      // errors.
    }
    callbacks.recordAnalysisErrors(path, convertedErrors);
  }

  /// Use the given analysis [driver] to analyze the content of the pubspec file
  /// at the given [path].
  void _analyzePubspecYaml(AnalysisDriver driver, String path) {
    var convertedErrors = const <protocol.AnalysisError>[];
    try {
      var content = _readFile(path);
      var node = loadYamlNode(content);
      if (node is YamlMap) {
        var validator = PubspecValidator(
            resourceProvider, resourceProvider.getFile(path).createSource());
        var lineInfo = _computeLineInfo(content);
        var errors = validator.validate(node.nodes);
        var converter = AnalyzerConverter();
        convertedErrors = converter.convertAnalysisErrors(errors,
            lineInfo: lineInfo, options: driver.analysisOptions);

        if (driver.analysisOptions.lint) {
          var visitors = <LintRule, PubspecVisitor>{};
          for (var linter in driver.analysisOptions.lintRules) {
            if (linter is LintRule) {
              var visitor = linter.getPubspecVisitor();
              if (visitor != null) {
                visitors[linter] = visitor;
              }
            }
          }

          if (visitors.isNotEmpty) {
            var sourceUri = resourceProvider.pathContext.toUri(path);
            var pubspecAst = Pubspec.parse(content,
                sourceUrl: sourceUri, resourceProvider: resourceProvider);
            var listener = RecordingErrorListener();
            var reporter = ErrorReporter(listener,
                resourceProvider.getFile(path).createSource(sourceUri),
                isNonNullableByDefault: false);
            for (var entry in visitors.entries) {
              entry.key.reporter = reporter;
              pubspecAst.accept(entry.value);
            }
            if (listener.errors.isNotEmpty) {
              convertedErrors.addAll(converter.convertAnalysisErrors(
                  listener.errors,
                  lineInfo: lineInfo,
                  options: driver.analysisOptions));
            }
          }
        }
      }
    } catch (exception) {
      // If the file cannot be analyzed, fall through to clear any previous
      // errors.
    }
    callbacks.recordAnalysisErrors(path, convertedErrors);
  }

  void _checkForAndroidManifestXmlUpdate(String path) {
    if (file_paths.isAndroidManifestXml(pathContext, path)) {
      var driver = getDriverFor(path);
      if (driver != null) {
        _analyzeAndroidManifestXml(driver, path);
      }
    }
  }

  void _checkForFixDataYamlUpdate(String path) {
    if (file_paths.isFixDataYaml(pathContext, path)) {
      var driver = getDriverFor(path);
      if (driver != null) {
        _analyzeFixDataYaml(driver, path);
      }
    }
  }

  /// Compute line information for the given [content].
  LineInfo _computeLineInfo(String content) {
    var lineStarts = StringUtilities.computeLineStarts(content);
    return LineInfo(lineStarts);
  }

  void _createAnalysisContexts() {
    _destroyAnalysisContexts();

    var collection = _collection = AnalysisContextCollectionImpl(
      includedPaths: includedPaths,
      excludedPaths: excludedPaths,
      byteStore: _byteStore,
      drainStreams: false,
      enableIndex: true,
      performanceLog: _performanceLog,
      resourceProvider: resourceProvider,
      scheduler: _scheduler,
      sdkPath: sdkManager.defaultSdkDirectory,
    );

    for (var analysisContext in collection.contexts) {
      var driver = analysisContext.driver;

      callbacks.listenAnalysisDriver(driver);

      var rootFolder = analysisContext.contextRoot.root;
      driverMap[rootFolder] = driver;

      changeSubscriptions[rootFolder] = rootFolder.changes
          .listen(_handleWatchEvent, onError: _handleWatchInterruption);

      _watchBazelFilesIfNeeded(rootFolder, driver);

      for (var file in analysisContext.contextRoot.analyzedFiles()) {
        if (file_paths.isAndroidManifestXml(pathContext, file)) {
          _analyzeAndroidManifestXml(driver, file);
        } else if (file_paths.isDart(pathContext, file)) {
          driver.addFile(file);
        }
      }

      var optionsFile = analysisContext.contextRoot.optionsFile;
      if (optionsFile != null) {
        _analyzeAnalysisOptionsYaml(driver, optionsFile.path);
      }

      var fixDataYamlFile = rootFolder
          .getChildAssumingFolder('lib')
          .getChildAssumingFile(file_paths.fixDataYaml);
      if (fixDataYamlFile.exists) {
        _analyzeFixDataYaml(driver, fixDataYamlFile.path);
      }

      var pubspecFile = rootFolder.getChildAssumingFile(file_paths.pubspecYaml);
      if (pubspecFile.exists) {
        _analyzePubspecYaml(driver, pubspecFile.path);
      }
    }

    callbacks.afterContextsCreated();
  }

  /// Clean up and destroy the context associated with the given folder.
  void _destroyAnalysisContext(DriverBasedAnalysisContext context) {
    context.driver.dispose();
    var rootFolder = context.contextRoot.root;
    changeSubscriptions.remove(rootFolder)?.cancel();
    var watched = bazelWatchedPathsPerFolder.remove(rootFolder);
    if (watched != null) {
      for (var path in watched.paths) {
        bazelWatcherService!.stopWatching(watched.workspace, path);
      }
      _stopWatchingBazelBinPaths(watched);
    }
    bazelSearchSubscriptions.remove(rootFolder)?.cancel();
    driverMap.remove(rootFolder);
  }

  void _destroyAnalysisContexts() {
    var collection = _collection;
    if (collection != null) {
      for (var analysisContext in collection.contexts) {
        _destroyAnalysisContext(analysisContext);
      }
      callbacks.afterContextsDestroyed();
    }
  }

  List<String> _getPossibelBazelBinPaths(_BazelWatchedFiles watched) => [
        pathContext.join(watched.workspace, 'bazel-bin'),
        pathContext.join(watched.workspace, 'blaze-bin'),
      ];

  /// Establishes watch(es) for the Bazel generated files provided in
  /// [notification].
  ///
  /// Whenever the files change, we trigger re-analysis. This allows us to react
  /// to creation/modification of files that were generated by Bazel.
  void _handleBazelSearchInfo(
      Folder folder, String workspace, BazelSearchInfo info) {
    final bazelWatcherService = this.bazelWatcherService;
    if (bazelWatcherService == null) {
      return;
    }

    var watched = bazelWatchedPathsPerFolder.putIfAbsent(
        folder, () => _BazelWatchedFiles(workspace));
    var added = watched.paths.add(info.requestedPath);
    if (added) bazelWatcherService.startWatching(workspace, info);
  }

  /// Notifies the drivers that a generated Bazel file has changed.
  void _handleBazelWatchEvents(List<WatchEvent> allEvents) {
    // First check if we have any changes to the bazel-*/blaze-* paths.  If
    // we do, we'll simply recreate all contexts to make sure that we follow the
    // correct paths.
    var bazelSymlinkPaths = bazelWatchedPathsPerFolder.values
        .expand((watched) => _getPossibelBazelBinPaths(watched))
        .toSet();
    if (allEvents.any((event) => bazelSymlinkPaths.contains(event.path))) {
      refresh();
      return;
    }

    var fileEvents =
        allEvents.where((event) => !bazelSymlinkPaths.contains(event.path));
    for (var driver in driverMap.values) {
      var needsUriReset = false;
      for (var event in fileEvents) {
        if (event.type == ChangeType.ADD) {
          driver.addFile(event.path);
          needsUriReset = true;
        }
        if (event.type == ChangeType.MODIFY) driver.changeFile(event.path);
        if (event.type == ChangeType.REMOVE) driver.removeFile(event.path);
      }
      // Since the file has been created after we've searched for it, the
      // URI resolution is likely wrong, so we need to reset it.
      if (needsUriReset) driver.resetUriResolution();
    }
  }

  void _handleWatchEvent(WatchEvent event) {
    callbacks.broadcastWatchEvent(event);
    _handleWatchEventImpl(event);
    callbacks.afterWatchEvent(event);
  }

  void _handleWatchEventImpl(WatchEvent event) {
    // Figure out which context this event applies to.
    // TODO(brianwilkerson) If a file is explicitly included in one context
    // but implicitly referenced in another context, we will only send a
    // changeSet to the context that explicitly includes the file (because
    // that's the only context that's watching the file).
    var path = event.path;
    var type = event.type;

    _instrumentationService.logWatchEvent('<unknown>', path, type.toString());

    if (file_paths.isAnalysisOptionsYaml(pathContext, path) ||
        file_paths.isDotPackages(pathContext, path) ||
        file_paths.isPackageConfigJson(pathContext, path) ||
        file_paths.isPubspecYaml(pathContext, path) ||
        false) {
      _createAnalysisContexts();
      return;
    }

    var collection = _collection;
    if (collection != null && file_paths.isDart(pathContext, path)) {
      for (var analysisContext in collection.contexts) {
        switch (type) {
          case ChangeType.ADD:
            if (analysisContext.contextRoot.isAnalyzed(path)) {
              analysisContext.driver.addFile(path);
            } else {
              analysisContext.driver.changeFile(path);
            }
            break;
          case ChangeType.MODIFY:
            analysisContext.driver.changeFile(path);
            break;
          case ChangeType.REMOVE:
            analysisContext.driver.removeFile(path);
            break;
        }
      }
    }

    switch (type) {
      case ChangeType.ADD:
      case ChangeType.MODIFY:
        _checkForAndroidManifestXmlUpdate(path);
        _checkForFixDataYamlUpdate(path);
        break;
      case ChangeType.REMOVE:
        callbacks.applyFileRemoved(path);
        break;
    }
  }

  /// On windows, the directory watcher may overflow, and we must recover.
  void _handleWatchInterruption(dynamic error, StackTrace stackTrace) {
    // We've handled the error, so we only have to log it.
    _instrumentationService
        .logError('Watcher error; refreshing contexts.\n$error\n$stackTrace');
    // TODO(mfairhurst): Optimize this, or perhaps be less complete.
    refresh();
  }

  /// Read the contents of the file at the given [path], or throw an exception
  /// if the contents cannot be read.
  String _readFile(String path) {
    return resourceProvider.getFile(path).readAsStringSync();
  }

  /// Checks whether the current roots were built using the same paths as
  /// [includedPaths]/[excludedPaths].
  bool _rootsAreUnchanged(
      List<String> includedPaths, List<String> excludedPaths) {
    if (includedPaths.length != this.includedPaths.length ||
        excludedPaths.length != this.excludedPaths.length) {
      return false;
    }
    final existingIncludedSet = this.includedPaths.toSet();
    final existingExcludedSet = this.excludedPaths.toSet();

    return existingIncludedSet.containsAll(includedPaths) &&
        existingExcludedSet.containsAll(excludedPaths);
  }

  /// Starts watching for the `bazel-bin` and `blaze-bin` symlinks.
  ///
  /// This is important since these symlinks might not be present when the
  /// server starts up, in which case `BazelWorkspace` assumes by default the
  /// Bazel ones.  So we want to detect if the symlinks get created to reset
  /// everything and repeat the search for the folders.
  void _startWatchingBazelBinPaths(_BazelWatchedFiles watched) {
    var watcherService = bazelWatcherService;
    if (watcherService == null) return;
    var paths = _getPossibelBazelBinPaths(watched);
    watcherService.startWatching(
        watched.workspace, BazelSearchInfo(paths[0], paths));
  }

  /// Stops watching for the `bazel-bin` and `blaze-bin` symlinks.
  void _stopWatchingBazelBinPaths(_BazelWatchedFiles watched) {
    var watcherService = bazelWatcherService;
    if (watcherService == null) return;
    var paths = _getPossibelBazelBinPaths(watched);
    watcherService.stopWatching(watched.workspace, paths[0]);
  }

  /// Listens to files generated by Bazel that were found or searched for.
  ///
  /// This is handled specially because the files are outside the package
  /// folder, but we still want to watch for changes to them.
  ///
  /// Does nothing if the [driver] is not in a Bazel workspace.
  void _watchBazelFilesIfNeeded(Folder folder, AnalysisDriver analysisDriver) {
    if (!experimentalEnableBazelWatching) return;
    var watcherService = bazelWatcherService;
    if (watcherService == null) return;

    var workspace = analysisDriver.analysisContext?.contextRoot.workspace;
    if (workspace is BazelWorkspace &&
        !bazelSearchSubscriptions.containsKey(folder)) {
      bazelSearchSubscriptions[folder] = workspace.bazelCandidateFiles.listen(
          (notification) =>
              _handleBazelSearchInfo(folder, workspace.root, notification));

      var watched = _BazelWatchedFiles(workspace.root);
      bazelWatchedPathsPerFolder[folder] = watched;
      _startWatchingBazelBinPaths(watched);
    }
  }
}

class NoopContextManagerCallbacks implements ContextManagerCallbacks {
  @override
  void afterContextsCreated() {}

  @override
  void afterContextsDestroyed() {}

  @override
  void afterWatchEvent(WatchEvent event) {}

  @override
  void applyFileRemoved(String file) {}

  @override
  void broadcastWatchEvent(WatchEvent event) {}

  @override
  void listenAnalysisDriver(AnalysisDriver driver) {}

  @override
  void recordAnalysisErrors(String path, List<protocol.AnalysisError> errors) {}
}

class _BazelWatchedFiles {
  final String workspace;
  final paths = <String>{};
  _BazelWatchedFiles(this.workspace);
}