// Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

library fasta.incremental_compiler;

import 'dart:async' show Future;

import 'package:front_end/src/api_prototype/experimental_flags.dart';
import 'package:kernel/binary/ast_from_binary.dart'
    show
        BinaryBuilderWithMetadata,
        CanonicalNameError,
        CanonicalNameSdkError,
        CompilationModeError,
        InvalidKernelSdkVersionError,
        InvalidKernelVersionError,
        SubComponentView;

import 'package:kernel/class_hierarchy.dart'
    show ClassHierarchy, ClosedWorldClassHierarchy;

import 'package:kernel/core_types.dart' show CoreTypes;

import 'package:kernel/kernel.dart'
    show
        Class,
        Component,
        DartType,
        Expression,
        FunctionNode,
        Library,
        LibraryDependency,
        LibraryPart,
        Name,
        NonNullableByDefaultCompiledMode,
        Procedure,
        ProcedureKind,
        ReturnStatement,
        Source,
        Supertype,
        TreeNode,
        TypeParameter;

import 'package:kernel/kernel.dart' as kernel show Combinator;

import 'package:kernel/target/changed_structure_notifier.dart'
    show ChangedStructureNotifier;

import 'package:package_config/package_config.dart' show Package, PackageConfig;

import '../api_prototype/file_system.dart' show FileSystem, FileSystemEntity;

import '../api_prototype/incremental_kernel_generator.dart'
    show IncrementalKernelGenerator, isLegalIdentifier;

import '../api_prototype/memory_file_system.dart' show MemoryFileSystem;

import 'builder/builder.dart' show Builder;

import 'builder/class_builder.dart' show ClassBuilder;

import 'builder/field_builder.dart' show FieldBuilder;

import 'builder/library_builder.dart' show LibraryBuilder;

import 'builder/name_iterator.dart' show NameIterator;

import 'builder/type_builder.dart' show TypeBuilder;

import 'builder/type_declaration_builder.dart' show TypeDeclarationBuilder;

import 'builder_graph.dart' show BuilderGraph;

import 'combinator.dart' show Combinator;

import 'compiler_context.dart' show CompilerContext;

import 'dill/dill_class_builder.dart' show DillClassBuilder;

import 'dill/dill_library_builder.dart' show DillLibraryBuilder;

import 'dill/dill_target.dart' show DillTarget;

import 'export.dart' show Export;

import 'import.dart' show Import;

import 'incremental_serializer.dart' show IncrementalSerializer;

import 'scope.dart' show Scope;

import 'source/source_class_builder.dart' show SourceClassBuilder;

import 'util/error_reporter_file_copier.dart' show saveAsGzip;

import 'util/experiment_environment_getter.dart' show getExperimentEnvironment;

import 'util/textual_outline.dart' show textualOutline;

import 'fasta_codes.dart'
    show
        DiagnosticMessageFromJson,
        templateInitializeFromDillNotSelfContained,
        templateInitializeFromDillNotSelfContainedNoDump,
        templateInitializeFromDillUnknownProblem,
        templateInitializeFromDillUnknownProblemNoDump;

import 'hybrid_file_system.dart' show HybridFileSystem;

import 'kernel/kernel_builder.dart' show ClassHierarchyBuilder;

import 'kernel/internal_ast.dart' show VariableDeclarationImpl;

import 'kernel/kernel_target.dart' show KernelTarget;

import 'library_graph.dart' show LibraryGraph;

import 'messages.dart' show Message;

import 'source/source_library_builder.dart' show SourceLibraryBuilder;

import 'ticker.dart' show Ticker;

import 'uri_translator.dart' show UriTranslator;

class IncrementalCompiler implements IncrementalKernelGenerator {
  final CompilerContext context;

  final Ticker ticker;
  final bool resetTicker;
  final bool outlineOnly;
  bool trackNeededDillLibraries = false;
  Set<Library> neededDillLibraries;

  Set<Uri> invalidatedUris = new Set<Uri>();

  DillTarget dillLoadedData;
  List<LibraryBuilder> platformBuilders;
  Map<Uri, LibraryBuilder> userBuilders;
  final Uri initializeFromDillUri;
  Component componentToInitializeFrom;
  bool initializedFromDill = false;
  bool initializedIncrementalSerializer = false;
  Uri previousPackagesUri;
  Map<String, Package> previousPackagesMap;
  Map<String, Package> currentPackagesMap;
  bool hasToCheckPackageUris = false;
  final bool initializedForExpressionCompilationOnly;
  bool computeDeltaRunOnce = false;
  Map<Uri, List<DiagnosticMessageFromJson>> remainingComponentProblems =
      new Map<Uri, List<DiagnosticMessageFromJson>>();
  List<Component> modulesToLoad;
  IncrementalSerializer incrementalSerializer;

  static final Uri debugExprUri =
      new Uri(scheme: "org-dartlang-debug", path: "synthetic_debug_expression");

  IncrementalKernelTarget userCode;
  Set<Library> previousSourceBuilders;

  IncrementalCompiler.fromComponent(
      this.context, this.componentToInitializeFrom,
      [bool outlineOnly, this.incrementalSerializer])
      : ticker = context.options.ticker,
        resetTicker = true,
        initializeFromDillUri = null,
        this.outlineOnly = outlineOnly ?? false,
        this.initializedForExpressionCompilationOnly = false {
    enableExperimentsBasedOnEnvironment();
  }

  IncrementalCompiler(this.context,
      [this.initializeFromDillUri,
      bool outlineOnly,
      this.incrementalSerializer])
      : ticker = context.options.ticker,
        resetTicker = true,
        componentToInitializeFrom = null,
        this.outlineOnly = outlineOnly ?? false,
        this.initializedForExpressionCompilationOnly = false {
    enableExperimentsBasedOnEnvironment();
  }

  IncrementalCompiler.forExpressionCompilationOnly(
      this.context, this.componentToInitializeFrom,
      [bool resetTicker])
      : ticker = context.options.ticker,
        this.resetTicker = resetTicker ?? true,
        initializeFromDillUri = null,
        this.outlineOnly = false,
        this.incrementalSerializer = null,
        this.initializedForExpressionCompilationOnly = true {
    enableExperimentsBasedOnEnvironment();
  }

  void enableExperimentsBasedOnEnvironment({Set<String> enabledExperiments}) {
    // Note that these are all experimental. Use at your own risk.
    enabledExperiments ??= getExperimentEnvironment();
    // Currently there's no live experiments.
  }

  @override
  void setExperimentalFeaturesForTesting(Set<String> features) {
    enableExperimentsBasedOnEnvironment(enabledExperiments: features);
  }

  @override
  Future<Component> computeDelta(
      {List<Uri> entryPoints, bool fullComponent: false}) async {
    if (resetTicker) {
      ticker.reset();
    }
    entryPoints ??= context.options.inputs;
    return context.runInContext<Component>((CompilerContext c) async {
      if (computeDeltaRunOnce && initializedForExpressionCompilationOnly) {
        throw new StateError("Initialized for expression compilation: "
            "cannot do another general compile.");
      }
      computeDeltaRunOnce = true;
      // Initial setup: Load platform, initialize from dill or component etc.
      UriTranslator uriTranslator = await setupPackagesAndUriTranslator(c);
      IncrementalCompilerData data =
          await ensurePlatformAndInitialize(uriTranslator, c);

      // Figure out what to keep and what to throw away.
      Set<Uri> invalidatedUris = this.invalidatedUris.toSet();
      invalidateNotKeptUserBuilders(invalidatedUris);
      ReusageResult reusedResult =
          computeReusedLibraries(invalidatedUris, uriTranslator);

      // Experimental invalidation initialization (e.g. figure out if we can).
      ExperimentalInvalidation experimentalInvalidation =
          await initializeExperimentalInvalidation(reusedResult, c);
      recordRebuildBodiesCountForTesting(
          experimentalInvalidation?.missingSources?.length ?? 0);

      // Cleanup: After (potentially) removing builders we have stuff to cleanup
      // to not leak, and we might need to re-create the dill target.
      cleanupRemovedBuilders(reusedResult, uriTranslator);
      recreateDillTargetIfPackageWasUpdated(uriTranslator, c);
      ClassHierarchy hierarchy = userCode?.loader?.hierarchy;
      cleanupHierarchy(hierarchy, experimentalInvalidation, reusedResult);
      List<LibraryBuilder> reusedLibraries = reusedResult.reusedLibraries;
      reusedResult = null;

      if (userCode != null) {
        ticker.logMs("Decided to reuse ${reusedLibraries.length}"
            " of ${userCode.loader.builders.length} libraries");
      }

      // For modular compilation we can be asked to load components and track
      // which libraries we actually use for the compilation. Set that up now.
      await loadEnsureLoadedComponents(reusedLibraries);
      resetTrackingOfUsedLibraries(hierarchy);

      // For each computeDelta call we create a new userCode object which needs
      // to be setup, and in the case of experimental invalidation some of the
      // builders needs to be patched up.
      KernelTarget userCodeOld = userCode;
      setupNewUserCode(c, uriTranslator, hierarchy, reusedLibraries,
          experimentalInvalidation, entryPoints.first);
      Map<LibraryBuilder, List<LibraryBuilder>> rebuildBodiesMap =
          experimentalInvalidationCreateRebuildBodiesBuilders(
              experimentalInvalidation, uriTranslator);
      entryPoints = userCode.setEntryPoints(entryPoints);
      await userCode.loader.buildOutlines();
      experimentalInvalidationPatchUpScopes(
          experimentalInvalidation, rebuildBodiesMap);
      rebuildBodiesMap = null;

      // Checkpoint: Build the actual outline.
      // Note that the [Component] is not the "full" component.
      // It is a component consisting of all newly compiled libraries and all
      // libraries loaded from .dill files or directly from components.
      // Technically, it's the combination of userCode.loader.libraries and
      // dillLoadedData.loader.libraries.
      Component componentWithDill = await userCode.buildOutlines();

      if (!outlineOnly) {
        // Checkpoint: Build the actual bodies.
        componentWithDill =
            await userCode.buildComponent(verify: c.options.verify);
      }
      hierarchy ??= userCode.loader.hierarchy;
      if (hierarchy != null) {
        if (userCode.classHierarchyChanges != null) {
          hierarchy.applyTreeChanges([], [], userCode.classHierarchyChanges);
        }
        if (userCode.classMemberChanges != null) {
          hierarchy.applyMemberChanges(userCode.classMemberChanges,
              findDescendants: true);
        }
      }
      recordNonFullComponentForTesting(componentWithDill);

      // Perform actual dill usage tracking.
      performDillUsageTracking(hierarchy);

      // If we actually got a result we can throw away the old userCode and the
      // list of invalidated uris.
      if (componentWithDill != null) {
        this.invalidatedUris.clear();
        hasToCheckPackageUris = false;
        userCodeOld?.loader?.releaseAncillaryResources();
        userCodeOld = null;
      }

      // Compute which libraries to output and which (previous) errors/warnings
      // we have to reissue. In the process do some cleanup too.
      List<Library> compiledLibraries =
          new List<Library>.from(userCode.loader.libraries);
      Map<Uri, Source> uriToSource = componentWithDill?.uriToSource;
      experimentalCompilationPostCompilePatchup(
          experimentalInvalidation, compiledLibraries, uriToSource);
      List<Library> outputLibraries =
          calculateOutputLibrariesAndIssueLibraryProblems(
              data.component != null || fullComponent,
              compiledLibraries,
              entryPoints,
              reusedLibraries,
              hierarchy,
              uriTranslator,
              uriToSource,
              c);
      List<String> problemsAsJson = reissueComponentProblems(componentWithDill);

      // If we didn't get a result, go back to the previous one so expression
      // calculation has the potential to work.
      if (componentWithDill == null) {
        userCode.loader.builders.clear();
        userCode = userCodeOld;
        dillLoadedData.loader.currentSourceLoader = userCode.loader;
      } else {
        previousSourceBuilders =
            await convertSourceLibraryBuildersToDill(experimentalInvalidation);
      }

      experimentalInvalidation = null;

      // Output result.
      Procedure mainMethod = componentWithDill == null
          ? data.component?.mainMethod
          : componentWithDill.mainMethod;
      NonNullableByDefaultCompiledMode compiledMode = componentWithDill == null
          ? data.component?.mode
          : componentWithDill.mode;
      return context.options.target.configureComponent(
          new Component(libraries: outputLibraries, uriToSource: uriToSource))
        ..setMainMethodAndMode(mainMethod?.reference, true, compiledMode)
        ..problemsAsJson = problemsAsJson;
    });
  }

  /// Convert every SourceLibraryBuilder to a DillLibraryBuilder.
  /// As we always do this, this will only be the new ones.
  ///
  /// If doing experimental invalidation that means that some of the old dill
  /// library builders might have links (via export scopes) to the
  /// source builders and they will thus be patched up here too.
  ///
  /// Returns the set of Libraries that now has new (dill) builders.
  Future<Set<Library>> convertSourceLibraryBuildersToDill(
      ExperimentalInvalidation experimentalInvalidation) async {
    bool changed = false;
    Set<Library> newDillLibraryBuilders = new Set<Library>();
    userBuilders ??= <Uri, LibraryBuilder>{};
    Map<LibraryBuilder, List<LibraryBuilder>> convertedLibraries;
    for (MapEntry<Uri, LibraryBuilder> entry
        in userCode.loader.builders.entries) {
      if (entry.value is SourceLibraryBuilder) {
        SourceLibraryBuilder builder = entry.value;
        DillLibraryBuilder dillBuilder =
            dillLoadedData.loader.appendLibrary(builder.library);
        userCode.loader.builders[entry.key] = dillBuilder;
        userBuilders[entry.key] = dillBuilder;
        newDillLibraryBuilders.add(builder.library);
        if (userCode.loader.first == builder) {
          userCode.loader.first = dillBuilder;
        }
        changed = true;
        if (experimentalInvalidation != null) {
          convertedLibraries ??=
              new Map<LibraryBuilder, List<LibraryBuilder>>();
          convertedLibraries[builder] = [dillBuilder];
        }
      }
    }
    if (changed) {
      // We suppress finalization errors because they have already been
      // reported.
      await dillLoadedData.buildOutlines(suppressFinalizationErrors: true);
      assert(_checkEquivalentScopes(
          userCode.loader.builders, dillLoadedData.loader.builders));

      if (experimentalInvalidation != null) {
        /// If doing experimental invalidation that means that some of the old
        /// dill library builders might have links (via export scopes) to the
        /// source builders. Patch that up.

        // Maps from old library builder to map of new content.
        Map<LibraryBuilder, Map<String, Builder>> replacementMap = {};

        // Maps from old library builder to map of new content.
        Map<LibraryBuilder, Map<String, Builder>> replacementSettersMap = {};

        experimentalInvalidationFillReplacementMaps(
            convertedLibraries, replacementMap, replacementSettersMap);

        for (LibraryBuilder builder
            in experimentalInvalidation.originalNotReusedLibraries) {
          DillLibraryBuilder dillBuilder = builder;
          if (dillBuilder.isBuilt) {
            dillBuilder.exportScope
                .patchUpScope(replacementMap, replacementSettersMap);

            // Clear cached calculations that points (potential) to now replaced
            // things.
            for (Builder builder in dillBuilder.scope.localMembers) {
              if (builder is DillClassBuilder) {
                builder.clearCachedValues();
              }
            }
          }
        }
        replacementMap = null;
        replacementSettersMap = null;
      }
    }
    userCode.loader.buildersCreatedWithReferences.clear();
    userCode.loader.builderHierarchy.clear();
    userCode.loader.referenceFromIndex = null;
    convertedLibraries = null;
    experimentalInvalidation = null;
    if (userBuilders.isEmpty) userBuilders = null;
    return newDillLibraryBuilders;
  }

  bool _checkEquivalentScopes(Map<Uri, LibraryBuilder> sourceLibraries,
      Map<Uri, LibraryBuilder> dillLibraries) {
    sourceLibraries.forEach((Uri uri, LibraryBuilder sourceLibraryBuilder) {
      if (sourceLibraryBuilder is SourceLibraryBuilder) {
        DillLibraryBuilder dillLibraryBuilder = dillLibraries[uri];
        assert(
            _hasEquivalentScopes(sourceLibraryBuilder, dillLibraryBuilder) ==
                null,
            _hasEquivalentScopes(sourceLibraryBuilder, dillLibraryBuilder));
      }
    });
    return true;
  }

  String _hasEquivalentScopes(SourceLibraryBuilder sourceLibraryBuilder,
      DillLibraryBuilder dillLibraryBuilder) {
    bool isEquivalent = true;
    StringBuffer sb = new StringBuffer();
    sb.writeln('Mismatch on ${sourceLibraryBuilder.importUri}:');
    sourceLibraryBuilder.exportScope
        .forEachLocalMember((String name, Builder sourceBuilder) {
      Builder dillBuilder =
          dillLibraryBuilder.exportScope.lookupLocalMember(name, setter: false);
      if (dillBuilder == null) {
        if ((name == 'dynamic' || name == 'Never') &&
            sourceLibraryBuilder.importUri == Uri.parse('dart:core')) {
          // The source library builder for dart:core has synthetically
          // injected builders for `dynamic` and `Never` which do not have
          // corresponding classes in the AST.
          return;
        }
        sb.writeln('No dill builder for ${name}: $sourceBuilder');
        isEquivalent = false;
      }
    });
    dillLibraryBuilder.exportScope
        .forEachLocalMember((String name, Builder dillBuilder) {
      Builder sourceBuilder = sourceLibraryBuilder.exportScope
          .lookupLocalMember(name, setter: false);
      if (sourceBuilder == null) {
        sb.writeln('No source builder for ${name}: $dillBuilder');
        isEquivalent = false;
      }
    });
    sourceLibraryBuilder.exportScope
        .forEachLocalSetter((String name, Builder sourceBuilder) {
      Builder dillBuilder =
          dillLibraryBuilder.exportScope.lookupLocalMember(name, setter: true);
      if (dillBuilder == null) {
        sb.writeln('No dill builder for ${name}=: $sourceBuilder');
        isEquivalent = false;
      }
    });
    dillLibraryBuilder.exportScope
        .forEachLocalSetter((String name, Builder dillBuilder) {
      Builder sourceBuilder = sourceLibraryBuilder.exportScope
          .lookupLocalMember(name, setter: true);
      if (sourceBuilder == null) {
        sourceBuilder = sourceLibraryBuilder.exportScope
            .lookupLocalMember(name, setter: false);
        if (sourceBuilder is FieldBuilder && sourceBuilder.isAssignable) {
          // Assignable fields can be lowered into a getter and setter.
          return;
        }
        sb.writeln('No source builder for ${name}=: $dillBuilder');
        isEquivalent = false;
      }
    });
    if (isEquivalent) {
      return null;
    }
    return sb.toString();
  }

  /// Compute which libraries to output and which (previous) errors/warnings we
  /// have to reissue. In the process do some cleanup too.
  List<Library> calculateOutputLibrariesAndIssueLibraryProblems(
      bool fullComponent,
      List<Library> compiledLibraries,
      List<Uri> entryPoints,
      List<LibraryBuilder> reusedLibraries,
      ClassHierarchy hierarchy,
      UriTranslator uriTranslator,
      Map<Uri, Source> uriToSource,
      CompilerContext c) {
    List<Library> outputLibraries;
    Set<Library> allLibraries;
    if (fullComponent) {
      outputLibraries = computeTransitiveClosure(compiledLibraries, entryPoints,
          reusedLibraries, hierarchy, uriTranslator, uriToSource);
      allLibraries = outputLibraries.toSet();
      if (!c.options.omitPlatform) {
        for (int i = 0; i < platformBuilders.length; i++) {
          Library lib = platformBuilders[i].library;
          outputLibraries.add(lib);
        }
      }
    } else {
      outputLibraries = new List<Library>();
      allLibraries = computeTransitiveClosure(
              compiledLibraries,
              entryPoints,
              reusedLibraries,
              hierarchy,
              uriTranslator,
              uriToSource,
              outputLibraries)
          .toSet();
    }

    reissueLibraryProblems(allLibraries, compiledLibraries);
    return outputLibraries;
  }

  /// If doing experimental compilation, make sure [compiledLibraries] and
  /// [uriToSource] looks as they would have if we hadn't done experimental
  /// compilation, i.e. before this call [compiledLibraries] might only contain
  /// the single Library we compiled again, but after this call, it will also
  /// contain all the libraries that would normally have been recompiled.
  /// This might be a temporary thing, but we need to figure out if the VM
  /// can (always) work with only getting the actually rebuild stuff.
  void experimentalCompilationPostCompilePatchup(
      ExperimentalInvalidation experimentalInvalidation,
      List<Library> compiledLibraries,
      Map<Uri, Source> uriToSource) {
    if (experimentalInvalidation != null) {
      // uriToSources are created in the outline stage which we skipped for
      // some of the libraries.
      for (Uri uri in experimentalInvalidation.missingSources) {
        // TODO(jensj): KernelTargets "link" takes some "excludeSource"
        // setting into account.
        uriToSource[uri] = CompilerContext.current.uriToSource[uri];
      }
    }
  }

  /// Perform dill usage tracking if asked. Use the marking on dill builders as
  /// well as the class hierarchy to figure out which dill libraries was
  /// actually used by the compilation.
  void performDillUsageTracking(ClassHierarchy hierarchy) {
    if (trackNeededDillLibraries) {
      // Which dill builders were built?
      neededDillLibraries = new Set<Library>();
      for (LibraryBuilder builder in dillLoadedData.loader.builders.values) {
        if (builder is DillLibraryBuilder) {
          if (builder.isBuiltAndMarked) {
            neededDillLibraries.add(builder.library);
          }
        }
      }

      updateNeededDillLibrariesWithHierarchy(
          hierarchy, userCode.loader.builderHierarchy);
    }
  }

  /// Fill in the replacement maps that describe the replacements that need to
  /// happen because of experimental invalidation.
  void experimentalInvalidationFillReplacementMaps(
      Map<LibraryBuilder, List<LibraryBuilder>> rebuildBodiesMap,
      Map<LibraryBuilder, Map<String, Builder>> replacementMap,
      Map<LibraryBuilder, Map<String, Builder>> replacementSettersMap) {
    for (MapEntry<LibraryBuilder, List<LibraryBuilder>> entry
        in rebuildBodiesMap.entries) {
      Map<String, Builder> childReplacementMap = {};
      Map<String, Builder> childReplacementSettersMap = {};
      List<LibraryBuilder> builders = rebuildBodiesMap[entry.key];
      replacementMap[entry.key] = childReplacementMap;
      replacementSettersMap[entry.key] = childReplacementSettersMap;
      for (LibraryBuilder builder in builders) {
        NameIterator iterator = builder.nameIterator;
        while (iterator.moveNext()) {
          Builder childBuilder = iterator.current;
          String name = iterator.name;
          Map<String, Builder> map;
          if (childBuilder.isSetter) {
            map = childReplacementSettersMap;
          } else {
            map = childReplacementMap;
          }
          assert(
              !map.containsKey(name),
              "Unexpected double-entry for $name in ${builder.importUri} "
              "(org from ${entry.key.importUri}): $childBuilder and "
              "${map[name]}");
          map[name] = childBuilder;
        }
      }
    }
  }

  /// When doing experimental invalidation, we have some builders that needs to
  /// be rebuild special, namely they have to be [userCode.loader.read] with
  /// references from the original [Library] for things to work.
  Map<LibraryBuilder, List<LibraryBuilder>>
      experimentalInvalidationCreateRebuildBodiesBuilders(
          ExperimentalInvalidation experimentalInvalidation,
          UriTranslator uriTranslator) {
    // Any builder(s) in [rebuildBodies] should be semi-reused: Create source
    // builders based on the underlying libraries.
    // Maps from old library builder to list of new library builder(s).
    Map<LibraryBuilder, List<LibraryBuilder>> rebuildBodiesMap =
        new Map<LibraryBuilder, List<LibraryBuilder>>.identity();
    if (experimentalInvalidation != null) {
      for (LibraryBuilder library in experimentalInvalidation.rebuildBodies) {
        LibraryBuilder newBuilder = userCode.loader.read(library.importUri, -1,
            accessor: userCode.loader.first,
            fileUri: library.fileUri,
            referencesFrom: library.library);
        List<LibraryBuilder> builders = [newBuilder];
        rebuildBodiesMap[library] = builders;
        for (LibraryPart part in library.library.parts) {
          // We need to pass the reference to make any class, procedure etc
          // overwrite correctly, but the library itself should  not be
          // over written as the library for parts are temporary "fake"
          // libraries.
          Uri partUri = getPartUri(library.importUri, part);
          Uri fileUri =
              getPartFileUri(library.library.fileUri, part, uriTranslator);
          LibraryBuilder newPartBuilder = userCode.loader.read(partUri, -1,
              accessor: library,
              fileUri: fileUri,
              referencesFrom: library.library,
              referenceIsPartOwner: true);
          builders.add(newPartBuilder);
        }
      }
    }
    return rebuildBodiesMap;
  }

  /// When doing experimental invalidation we have to patch up the scopes of the
  /// the libraries we're not recompiling but should have recompiled if we
  /// didn't do anything special.
  void experimentalInvalidationPatchUpScopes(
      ExperimentalInvalidation experimentalInvalidation,
      Map<LibraryBuilder, List<LibraryBuilder>> rebuildBodiesMap) {
    if (experimentalInvalidation != null) {
      // Maps from old library builder to map of new content.
      Map<LibraryBuilder, Map<String, Builder>> replacementMap = {};

      // Maps from old library builder to map of new content.
      Map<LibraryBuilder, Map<String, Builder>> replacementSettersMap = {};

      experimentalInvalidationFillReplacementMaps(
          rebuildBodiesMap, replacementMap, replacementSettersMap);

      for (LibraryBuilder builder
          in experimentalInvalidation.originalNotReusedLibraries) {
        if (builder is SourceLibraryBuilder) {
          builder.clearExtensionsInScopeCache();
          for (Import import in builder.imports) {
            assert(import.importer == builder);
            List<LibraryBuilder> replacements =
                rebuildBodiesMap[import.imported];
            if (replacements != null) {
              import.imported = replacements.first;
            }
            if (import.prefixBuilder?.exportScope != null) {
              Scope scope = import.prefixBuilder?.exportScope;
              scope.patchUpScope(replacementMap, replacementSettersMap);
            }
          }
          for (Export export in builder.exports) {
            assert(export.exporter == builder);
            List<LibraryBuilder> replacements =
                rebuildBodiesMap[export.exported];

            if (replacements != null) {
              export.exported = replacements.first;
            }
          }
          builder.exportScope
              .patchUpScope(replacementMap, replacementSettersMap);
          builder.importScope
              .patchUpScope(replacementMap, replacementSettersMap);

          Iterator<Builder> iterator = builder.iterator;
          while (iterator.moveNext()) {
            Builder childBuilder = iterator.current;
            if (childBuilder is SourceClassBuilder) {
              TypeBuilder typeBuilder = childBuilder.supertypeBuilder;
              replaceTypeBuilder(
                  replacementMap, replacementSettersMap, typeBuilder);
              typeBuilder = childBuilder.mixedInTypeBuilder;
              replaceTypeBuilder(
                  replacementMap, replacementSettersMap, typeBuilder);
              if (childBuilder.onTypes != null) {
                for (typeBuilder in childBuilder.onTypes) {
                  replaceTypeBuilder(
                      replacementMap, replacementSettersMap, typeBuilder);
                }
              }
              if (childBuilder.interfaceBuilders != null) {
                for (typeBuilder in childBuilder.interfaceBuilders) {
                  replaceTypeBuilder(
                      replacementMap, replacementSettersMap, typeBuilder);
                }
              }
            }
          }
        } else if (builder is DillLibraryBuilder) {
          DillLibraryBuilder dillBuilder = builder;
          // There's only something to patch up if it was build already.
          if (dillBuilder.isBuilt) {
            dillBuilder.exportScope
                .patchUpScope(replacementMap, replacementSettersMap);
          }
        } else {
          throw new StateError(
              "Unexpected builder: $builder (${builder.runtimeType})");
        }
      }
    }
  }

  IncrementalKernelTarget createIncrementalKernelTarget(
      FileSystem fileSystem,
      bool includeComments,
      DillTarget dillTarget,
      UriTranslator uriTranslator) {
    return new IncrementalKernelTarget(
        fileSystem, includeComments, dillTarget, uriTranslator);
  }

  /// Create a new [userCode] object, and add the reused builders to it.
  void setupNewUserCode(
      CompilerContext c,
      UriTranslator uriTranslator,
      ClassHierarchy hierarchy,
      List<LibraryBuilder> reusedLibraries,
      ExperimentalInvalidation experimentalInvalidation,
      Uri firstEntryPoint) {
    userCode = createIncrementalKernelTarget(
        new HybridFileSystem(
            new MemoryFileSystem(
                new Uri(scheme: "org-dartlang-debug", path: "/")),
            c.fileSystem),
        false,
        dillLoadedData,
        uriTranslator);
    userCode.loader.hierarchy = hierarchy;
    dillLoadedData.loader.currentSourceLoader = userCode.loader;

    // Re-use the libraries we've deemed re-usable.
    for (LibraryBuilder library in reusedLibraries) {
      userCode.loader.builders[library.importUri] = library;
      if (library.importUri.scheme == "dart" &&
          library.importUri.path == "core") {
        userCode.loader.coreLibrary = library;
      }
    }

    // The entry point(s) has to be set first for loader.first to be setup
    // correctly. If the first one is in the rebuildBodies, we have to add it
    // from there first.
    Uri firstEntryPointImportUri =
        userCode.getEntryPointUri(firstEntryPoint, issueProblem: false);
    bool wasFirstSet = false;
    if (experimentalInvalidation != null) {
      for (LibraryBuilder library in experimentalInvalidation.rebuildBodies) {
        if (library.importUri == firstEntryPointImportUri) {
          userCode.loader.read(library.importUri, -1,
              accessor: userCode.loader.first,
              fileUri: library.fileUri,
              referencesFrom: library.library);
          wasFirstSet = true;
          break;
        }
      }
    }
    if (!wasFirstSet) {
      userCode.loader.read(firstEntryPointImportUri, -1,
          accessor: userCode.loader.first,
          fileUri: firstEntryPointImportUri != firstEntryPoint
              ? firstEntryPoint
              : null);
    }
    if (userCode.loader.first == null &&
        userCode.loader.builders[firstEntryPointImportUri] != null) {
      userCode.loader.first =
          userCode.loader.builders[firstEntryPointImportUri];
    }
  }

  /// When tracking used libraries we mark them when we use them. To track
  /// correctly we have to unmark before the next iteration to not have too much
  /// marked and therefore incorrectly marked something as used when it is not.
  void resetTrackingOfUsedLibraries(ClassHierarchy hierarchy) {
    if (trackNeededDillLibraries) {
      // Reset dill loaders and kernel class hierarchy.
      for (LibraryBuilder builder in dillLoadedData.loader.builders.values) {
        if (builder is DillLibraryBuilder) {
          if (builder.isBuiltAndMarked) {
            // Clear cached calculations in classes which upon calculation can
            // mark things as needed.
            for (Builder builder in builder.scope.localMembers) {
              if (builder is DillClassBuilder) {
                builder.clearCachedValues();
              }
            }
            builder.isBuiltAndMarked = false;
          }
        }
      }

      if (hierarchy is ClosedWorldClassHierarchy) {
        hierarchy.resetUsed();
      }
    }
  }

  /// Cleanup the hierarchy to no longer reference libraries that we are
  /// invalidating (or would normally have invalidated if we hadn't done any
  /// experimental invalidation).
  void cleanupHierarchy(
      ClassHierarchy hierarchy,
      ExperimentalInvalidation experimentalInvalidation,
      ReusageResult reusedResult) {
    if (hierarchy != null) {
      List<Library> removedLibraries = new List<Library>();
      // TODO(jensj): For now remove all the original from the class hierarchy
      // to avoid the class hierarchy getting confused.
      if (experimentalInvalidation != null) {
        for (LibraryBuilder builder
            in experimentalInvalidation.originalNotReusedLibraries) {
          Library lib = builder.library;
          removedLibraries.add(lib);
        }
      }
      for (LibraryBuilder builder in reusedResult.notReusedLibraries) {
        Library lib = builder.library;
        removedLibraries.add(lib);
      }
      hierarchy.applyTreeChanges(removedLibraries, const [], const []);
    }
  }

  /// If the package uris needs to be re-checked the uri translator has changed,
  /// and the [DillTarget] needs to get the new uri translator. We do that
  /// by creating a new one.
  void recreateDillTargetIfPackageWasUpdated(
      UriTranslator uriTranslator, CompilerContext c) {
    if (hasToCheckPackageUris) {
      // The package file was changed.
      // Make sure the dill loader is on the same page.
      DillTarget oldDillLoadedData = dillLoadedData;
      dillLoadedData = new DillTarget(ticker, uriTranslator, c.options.target);
      for (DillLibraryBuilder library
          in oldDillLoadedData.loader.builders.values) {
        library.loader = dillLoadedData.loader;
        dillLoadedData.loader.builders[library.importUri] = library;
        if (library.importUri.scheme == "dart" &&
            library.importUri.path == "core") {
          dillLoadedData.loader.coreLibrary = library;
        }
      }
      dillLoadedData.loader.first = oldDillLoadedData.loader.first;
      dillLoadedData.loader.libraries
          .addAll(oldDillLoadedData.loader.libraries);
    }
  }

  /// Builders we don't use again should be removed from places like
  /// uriToSource (used in places for dependency tracking), the incremental
  /// serializer (they are no longer kept up-to-date) and the DillTarget
  /// (to avoid leaks).
  /// We also have to remove any component problems beloning to any such
  /// no-longer-used library (to avoid re-issuing errors about no longer
  /// relevant stuff).
  void cleanupRemovedBuilders(
      ReusageResult reusedResult, UriTranslator uriTranslator) {
    bool removedDillBuilders = false;
    for (LibraryBuilder builder in reusedResult.notReusedLibraries) {
      cleanupSourcesForBuilder(
          builder, uriTranslator, CompilerContext.current.uriToSource);
      incrementalSerializer?.invalidate(builder.fileUri);

      LibraryBuilder dillBuilder =
          dillLoadedData.loader.builders.remove(builder.importUri);
      if (dillBuilder != null) {
        removedDillBuilders = true;
        userBuilders?.remove(builder.importUri);
      }

      // Remove component problems for libraries we don't reuse.
      if (remainingComponentProblems.isNotEmpty) {
        Library lib = builder.library;
        removeLibraryFromRemainingComponentProblems(lib, uriTranslator);
      }
    }

    if (removedDillBuilders) {
      makeDillLoaderLibrariesUpToDateWithBuildersMap();
    }
  }

  /// Figure out if we can (and was asked to) do experimental invalidation.
  /// Note that this returns (future or) [null] if we're not doing experimental
  /// invalidation.
  Future<ExperimentalInvalidation> initializeExperimentalInvalidation(
      ReusageResult reusedResult, CompilerContext c) async {
    Set<LibraryBuilder> rebuildBodies;
    Set<LibraryBuilder> originalNotReusedLibraries;
    Set<Uri> missingSources;

    if (!context.options.isExperimentEnabledGlobally(
        ExperimentalFlag.alternativeInvalidationStrategy)) return null;
    if (modulesToLoad != null) return null;
    if (reusedResult.directlyInvalidated.isEmpty) return null;
    if (reusedResult.invalidatedBecauseOfPackageUpdate) return null;

    // Figure out if the file(s) have changed outline, or we can just
    // rebuild the bodies.
    for (int i = 0; i < reusedResult.directlyInvalidated.length; i++) {
      LibraryBuilder builder = reusedResult.directlyInvalidated[i];
      if (builder.library.problemsAsJson != null) {
        assert(builder.library.problemsAsJson.isNotEmpty);
        return null;
      }
      Iterator<Builder> iterator = builder.iterator;
      while (iterator.moveNext()) {
        Builder childBuilder = iterator.current;
        if (childBuilder.isDuplicate) {
          return null;
        }
      }

      List<int> previousSource =
          CompilerContext.current.uriToSource[builder.fileUri].source;
      if (previousSource == null || previousSource.isEmpty) {
        return null;
      }
      String before = textualOutline(previousSource, performModelling: true);
      if (before == null) {
        return null;
      }
      String now;
      FileSystemEntity entity =
          c.options.fileSystem.entityForUri(builder.fileUri);
      if (await entity.exists()) {
        now =
            textualOutline(await entity.readAsBytes(), performModelling: true);
      }
      if (before != now) {
        return null;
      }
      // TODO(jensj): We should only do this when we're sure we're going to
      // do it!
      CompilerContext.current.uriToSource.remove(builder.fileUri);
      missingSources ??= new Set<Uri>();
      missingSources.add(builder.fileUri);
      LibraryBuilder partOfLibrary = builder.partOfLibrary;
      rebuildBodies ??= new Set<LibraryBuilder>();
      if (partOfLibrary != null) {
        rebuildBodies.add(partOfLibrary);
      } else {
        rebuildBodies.add(builder);
      }
    }

    // TODO(jensj): Check for mixins in a smarter and faster way.
    for (LibraryBuilder builder in reusedResult.notReusedLibraries) {
      if (missingSources.contains(builder.fileUri)) {
        continue;
      }
      Library lib = builder.library;
      for (Class c in lib.classes) {
        if (!c.isAnonymousMixin && !c.isEliminatedMixin) {
          continue;
        }
        for (Supertype supertype in c.implementedTypes) {
          if (missingSources.contains(supertype.classNode.fileUri)) {
            // This is probably a mixin from one of the libraries we want
            // to rebuild only the body of.
            // TODO(jensj): We can probably add this to the rebuildBodies
            // list and just rebuild that library too.
            // print("Usage of mixin in ${lib.importUri}");
            return null;
          }
        }
      }
    }

    originalNotReusedLibraries = new Set<LibraryBuilder>();
    Set<Uri> seenUris = new Set<Uri>();
    for (LibraryBuilder builder in reusedResult.notReusedLibraries) {
      if (builder.isPart) continue;
      if (builder.isPatch) continue;
      if (rebuildBodies.contains(builder)) continue;
      if (!seenUris.add(builder.importUri)) continue;
      reusedResult.reusedLibraries.add(builder);
      originalNotReusedLibraries.add(builder);
    }
    reusedResult.notReusedLibraries.clear();
    reusedResult.notReusedLibraries.addAll(rebuildBodies);

    return new ExperimentalInvalidation(
        rebuildBodies, originalNotReusedLibraries, missingSources);
  }

  /// Get UriTranslator, and figure out if the packages file was (potentially)
  /// changed.
  Future<UriTranslator> setupPackagesAndUriTranslator(CompilerContext c) async {
    bool bypassCache = false;
    if (!identical(previousPackagesUri, c.options.packagesUriRaw)) {
      previousPackagesUri = c.options.packagesUriRaw;
      bypassCache = true;
    } else if (this.invalidatedUris.contains(c.options.packagesUri)) {
      bypassCache = true;
    }
    UriTranslator uriTranslator =
        await c.options.getUriTranslator(bypassCache: bypassCache);
    previousPackagesMap = currentPackagesMap;
    currentPackagesMap = createPackagesMap(uriTranslator.packages);
    // TODO(jensj): We can probably (from the maps above) figure out if anything
    // changed and only set this to true if it did.
    hasToCheckPackageUris = hasToCheckPackageUris || bypassCache;
    ticker.logMs("Read packages file");
    if (initializedForExpressionCompilationOnly) {
      hasToCheckPackageUris = false;
    }
    return uriTranslator;
  }

  Map<String, Package> createPackagesMap(PackageConfig packages) {
    Map<String, Package> result = new Map<String, Package>();
    for (Package package in packages.packages) {
      result[package.name] = package;
    }
    return result;
  }

  /// Load platform and (potentially) initialize from dill,
  /// or initialize from component.
  Future<IncrementalCompilerData> ensurePlatformAndInitialize(
      UriTranslator uriTranslator, CompilerContext c) async {
    IncrementalCompilerData data = new IncrementalCompilerData();
    if (dillLoadedData == null) {
      int bytesLength = 0;
      if (componentToInitializeFrom != null) {
        // If initializing from a component it has to include the sdk,
        // so we explicitly don't load it here.
        initializeFromComponent(uriTranslator, c, data);
        componentToInitializeFrom = null;
      } else {
        List<int> summaryBytes = await c.options.loadSdkSummaryBytes();
        bytesLength = prepareSummary(summaryBytes, uriTranslator, c, data);
        if (initializeFromDillUri != null) {
          try {
            bytesLength += await initializeFromDill(uriTranslator, c, data);
          } catch (e, st) {
            // We might have loaded x out of y libraries into the component.
            // To avoid any unforeseen problems start over.
            bytesLength = prepareSummary(summaryBytes, uriTranslator, c, data);

            if (e is InvalidKernelVersionError ||
                e is InvalidKernelSdkVersionError ||
                e is PackageChangedError ||
                e is CanonicalNameSdkError ||
                e is CompilationModeError) {
              // Don't report any warning.
            } else {
              Uri gzInitializedFrom;
              if (c.options.writeFileOnCrashReport) {
                gzInitializedFrom = saveAsGzip(
                    data.initializationBytes, "initialize_from.dill");
                recordTemporaryFileForTesting(gzInitializedFrom);
              }
              if (e is CanonicalNameError) {
                Message message = gzInitializedFrom != null
                    ? templateInitializeFromDillNotSelfContained.withArguments(
                        initializeFromDillUri.toString(), gzInitializedFrom)
                    : templateInitializeFromDillNotSelfContainedNoDump
                        .withArguments(initializeFromDillUri.toString());
                dillLoadedData.loader
                    .addProblem(message, TreeNode.noOffset, 1, null);
              } else {
                // Unknown error: Report problem as such.
                Message message = gzInitializedFrom != null
                    ? templateInitializeFromDillUnknownProblem.withArguments(
                        initializeFromDillUri.toString(),
                        "$e",
                        "$st",
                        gzInitializedFrom)
                    : templateInitializeFromDillUnknownProblemNoDump
                        .withArguments(
                            initializeFromDillUri.toString(), "$e", "$st");
                dillLoadedData.loader
                    .addProblem(message, TreeNode.noOffset, 1, null);
              }
            }
          }
        }
      }
      appendLibraries(data, bytesLength);

      // We suppress finalization errors because they will reported via
      // problemsAsJson fields (with better precision).
      await dillLoadedData.buildOutlines(suppressFinalizationErrors: true);
      userBuilders = <Uri, LibraryBuilder>{};
      platformBuilders = <LibraryBuilder>[];
      dillLoadedData.loader.builders.forEach((uri, builder) {
        if (builder.importUri.scheme == "dart") {
          platformBuilders.add(builder);
        } else {
          userBuilders[uri] = builder;
        }
      });
      if (userBuilders.isEmpty) userBuilders = null;
    }
    data.initializationBytes = null;
    return data;
  }

  void replaceTypeBuilder(
      Map<LibraryBuilder, Map<String, Builder>> replacementMap,
      Map<LibraryBuilder, Map<String, Builder>> replacementSettersMap,
      TypeBuilder typeBuilder) {
    TypeDeclarationBuilder declaration = typeBuilder?.declaration;
    Builder parent = declaration?.parent;
    if (parent == null) return;
    Map<String, Builder> childReplacementMap;
    if (declaration.isSetter) {
      childReplacementMap = replacementSettersMap[parent];
    } else {
      childReplacementMap = replacementMap[parent];
    }

    if (childReplacementMap == null) return;
    Builder replacement = childReplacementMap[declaration.name];
    assert(replacement != null, "Didn't find the replacement for $typeBuilder");
    typeBuilder.bind(replacement);
  }

  @override
  CoreTypes getCoreTypes() => userCode?.loader?.coreTypes;

  @override
  ClassHierarchy getClassHierarchy() => userCode?.loader?.hierarchy;

  /// Allows for updating the list of needed libraries.
  ///
  /// Useful if a class hierarchy has been used externally.
  /// Currently there are two different class hierarchies which is unfortunate.
  /// For now this method allows the 'ClassHierarchyBuilder' to be null.
  ///
  /// TODO(jensj,CFE in general): Eventually we should get to a point where we
  /// only have one class hierarchy.
  /// TODO(jensj): This could probably be a utility method somewhere instead
  /// (though handling of the case where all bets are off should probably still
  /// live locally).
  void updateNeededDillLibrariesWithHierarchy(
      ClassHierarchy hierarchy, ClassHierarchyBuilder builderHierarchy) {
    if (hierarchy is ClosedWorldClassHierarchy && !hierarchy.allBetsOff) {
      neededDillLibraries ??= new Set<Library>();
      Set<Class> classes = new Set<Class>();
      List<Class> worklist = new List<Class>();
      // Get all classes touched by kernel class hierarchy.
      List<Class> usedClasses = hierarchy.getUsedClasses();
      worklist.addAll(usedClasses);
      classes.addAll(usedClasses);

      // Get all classes touched by fasta class hierarchy.
      if (builderHierarchy != null) {
        for (Class c in builderHierarchy.nodes.keys) {
          if (classes.add(c)) worklist.add(c);
        }
      }

      // Get all supers etc.
      while (worklist.isNotEmpty) {
        Class c = worklist.removeLast();
        for (Supertype supertype in c.implementedTypes) {
          if (classes.add(supertype.classNode)) {
            worklist.add(supertype.classNode);
          }
        }
        if (c.mixedInType != null) {
          if (classes.add(c.mixedInType.classNode)) {
            worklist.add(c.mixedInType.classNode);
          }
        }
        if (c.supertype != null) {
          if (classes.add(c.supertype.classNode)) {
            worklist.add(c.supertype.classNode);
          }
        }
      }

      // Add any libraries that was used or was in the "parent-chain" of a
      // used class.
      for (Class c in classes) {
        Library library = c.enclosingLibrary;
        // Only add if loaded from a dill file (and wasn't a 'dill' that was
        // converted from source builders to dill builders).
        if (dillLoadedData.loader.builders.containsKey(library.importUri) &&
            (previousSourceBuilders == null ||
                !previousSourceBuilders.contains(library))) {
          neededDillLibraries.add(library);
        }
      }
    } else {
      // Cannot track in other kernel class hierarchies or
      // if all bets are off: Add everything (except for the libraries we just
      // converted from source builders to dill builders).
      neededDillLibraries = new Set<Library>();
      for (LibraryBuilder builder in dillLoadedData.loader.builders.values) {
        if (builder is DillLibraryBuilder &&
            (previousSourceBuilders == null ||
                !previousSourceBuilders.contains(builder.library))) {
          neededDillLibraries.add(builder.library);
        }
      }
    }
  }

  /// Internal method.
  void invalidateNotKeptUserBuilders(Set<Uri> invalidatedUris) {
    if (modulesToLoad != null && userBuilders != null) {
      Set<Library> loadedNotKept = new Set<Library>();
      for (LibraryBuilder builder in userBuilders.values) {
        loadedNotKept.add(builder.library);
      }
      for (Component module in modulesToLoad) {
        loadedNotKept.removeAll(module.libraries);
      }
      for (Library lib in loadedNotKept) {
        invalidatedUris.add(lib.importUri);
      }
    }
  }

  /// Internal method.
  Future<void> loadEnsureLoadedComponents(
      List<LibraryBuilder> reusedLibraries) async {
    if (modulesToLoad != null) {
      bool loadedAnything = false;
      for (Component module in modulesToLoad) {
        bool usedComponent = false;
        for (Library lib in module.libraries) {
          if (!dillLoadedData.loader.builders.containsKey(lib.importUri)) {
            dillLoadedData.loader.libraries.add(lib);
            dillLoadedData.addLibrary(lib);
            reusedLibraries.add(dillLoadedData.loader.read(lib.importUri, -1));
            usedComponent = true;
          }
        }
        if (usedComponent) {
          dillLoadedData.uriToSource.addAll(module.uriToSource);
          loadedAnything = true;
        }
      }
      if (loadedAnything) {
        // We suppress finalization errors because they will reported via
        // problemsAsJson fields (with better precision).
        await dillLoadedData.buildOutlines(suppressFinalizationErrors: true);
        userBuilders = <Uri, LibraryBuilder>{};
        platformBuilders = <LibraryBuilder>[];
        dillLoadedData.loader.builders.forEach((uri, builder) {
          if (builder.importUri.scheme == "dart") {
            platformBuilders.add(builder);
          } else {
            userBuilders[uri] = builder;
          }
        });
        if (userBuilders.isEmpty) {
          userBuilders = null;
        }
      }
      modulesToLoad = null;
    }
  }

  /// Internal method.
  void reissueLibraryProblems(
      Set<Library> allLibraries, List<Library> compiledLibraries) {
    // The newly-compiled libraries have issued problems already. Re-issue
    // problems for the libraries that weren't re-compiled (ignore compile
    // expression problems)
    allLibraries.removeAll(compiledLibraries);
    for (Library library in allLibraries) {
      if (library.problemsAsJson?.isNotEmpty == true) {
        for (String jsonString in library.problemsAsJson) {
          DiagnosticMessageFromJson message =
              new DiagnosticMessageFromJson.fromJson(jsonString);
          if (message.uri == debugExprUri) {
            continue;
          }
          context.options.reportDiagnosticMessage(message);
        }
      }
    }
  }

  /// Internal method.
  /// Re-issue problems on the component and return the filtered list.
  List<String> reissueComponentProblems(Component componentWithDill) {
    // These problems have already been reported.
    Set<String> issuedProblems = new Set<String>();
    if (componentWithDill?.problemsAsJson != null) {
      issuedProblems.addAll(componentWithDill.problemsAsJson);
    }

    // Report old problems that wasn't reported again.
    for (List<DiagnosticMessageFromJson> messages
        in remainingComponentProblems.values) {
      for (int i = 0; i < messages.length; i++) {
        DiagnosticMessageFromJson message = messages[i];
        if (issuedProblems.add(message.toJsonString())) {
          context.options.reportDiagnosticMessage(message);
        }
      }
    }

    // Save any new component-problems.
    if (componentWithDill?.problemsAsJson != null) {
      for (String jsonString in componentWithDill.problemsAsJson) {
        DiagnosticMessageFromJson message =
            new DiagnosticMessageFromJson.fromJson(jsonString);
        List<DiagnosticMessageFromJson> messages =
            remainingComponentProblems[message.uri] ??=
                new List<DiagnosticMessageFromJson>();
        messages.add(message);
      }
    }
    return new List<String>.from(issuedProblems);
  }

  /// Internal method.
  Uri getPartFileUri(
      Uri parentFileUri, LibraryPart part, UriTranslator uriTranslator) {
    Uri fileUri = getPartUri(parentFileUri, part);
    if (fileUri.scheme == "package") {
      // Part was specified via package URI and the resolve above thus
      // did not go as expected. Translate the package URI to get the
      // actual file URI.
      fileUri = uriTranslator.translate(fileUri, false);
    }
    return fileUri;
  }

  /// Internal method.
  /// Compute the transitive closure.
  ///
  /// As a side-effect, this also cleans-up now-unreferenced builders as well as
  /// any saved component problems for such builders.
  List<Library> computeTransitiveClosure(
      List<Library> inputLibraries,
      List<Uri> entries,
      List<LibraryBuilder> reusedLibraries,
      ClassHierarchy hierarchy,
      UriTranslator uriTranslator,
      Map<Uri, Source> uriToSource,
      [List<Library> inputLibrariesFiltered]) {
    List<Library> result = new List<Library>();
    Map<Uri, Library> libraryMap = <Uri, Library>{};
    Map<Uri, Library> potentiallyReferencedLibraries = <Uri, Library>{};
    Map<Uri, Library> potentiallyReferencedInputLibraries = <Uri, Library>{};
    for (Library library in inputLibraries) {
      libraryMap[library.importUri] = library;
      if (library.importUri.scheme == "dart") {
        result.add(library);
        inputLibrariesFiltered?.add(library);
      } else {
        potentiallyReferencedLibraries[library.importUri] = library;
        potentiallyReferencedInputLibraries[library.importUri] = library;
      }
    }

    List<Uri> worklist = new List<Uri>();
    worklist.addAll(entries);
    for (LibraryBuilder libraryBuilder in reusedLibraries) {
      if (libraryBuilder.importUri.scheme == "dart" &&
          !libraryBuilder.isSynthetic) {
        continue;
      }
      Library lib = libraryBuilder.library;
      potentiallyReferencedLibraries[libraryBuilder.importUri] = lib;
      libraryMap[libraryBuilder.importUri] = lib;
    }

    LibraryGraph graph = new LibraryGraph(libraryMap);
    Set<Uri> partsUsed = new Set<Uri>();
    while (worklist.isNotEmpty && potentiallyReferencedLibraries.isNotEmpty) {
      Uri uri = worklist.removeLast();
      if (libraryMap.containsKey(uri)) {
        for (Uri neighbor in graph.neighborsOf(uri)) {
          worklist.add(neighbor);
        }
        libraryMap.remove(uri);
        Library library = potentiallyReferencedLibraries.remove(uri);
        if (library != null) {
          result.add(library);
          if (potentiallyReferencedInputLibraries.remove(uri) != null) {
            inputLibrariesFiltered?.add(library);
          }
          for (LibraryPart part in library.parts) {
            Uri partFileUri =
                getPartFileUri(library.fileUri, part, uriTranslator);
            partsUsed.add(partFileUri);
          }
          partsUsed;
        }
      }
    }

    List<Library> removedLibraries = new List<Library>();
    bool removedDillBuilders = false;
    for (Uri uri in potentiallyReferencedLibraries.keys) {
      if (uri.scheme == "package") continue;
      LibraryBuilder builder = userCode.loader.builders.remove(uri);
      if (builder != null) {
        Library lib = builder.library;
        removedLibraries.add(lib);
        if (dillLoadedData.loader.builders.remove(uri) != null) {
          removedDillBuilders = true;
        }
        cleanupSourcesForBuilder(builder, uriTranslator,
            CompilerContext.current.uriToSource, uriToSource, partsUsed);
        userBuilders?.remove(uri);
        removeLibraryFromRemainingComponentProblems(
            lib, uriTranslator, partsUsed);

        // Technically this isn't necessary as the uri is not a package-uri.
        incrementalSerializer?.invalidate(builder.fileUri);
      }
    }
    hierarchy?.applyTreeChanges(removedLibraries, const [], const []);
    if (removedDillBuilders) {
      makeDillLoaderLibrariesUpToDateWithBuildersMap();
    }

    return result;
  }

  /// If builders was removed from the [dillLoadedData.loader.builders] map
  /// the loaders [libraries] list has to be updated too, or those libraries
  /// will still hang around and be linked into the Component created internally
  /// in the compilation process.
  /// This method syncs the [libraries] list with the data in [builders].
  void makeDillLoaderLibrariesUpToDateWithBuildersMap() {
    dillLoadedData.loader.libraries.clear();
    for (LibraryBuilder builder in dillLoadedData.loader.builders.values) {
      dillLoadedData.loader.libraries.add(builder.library);
    }
  }

  /// Internal method.
  ///
  /// [partsUsed] indicates part uris that are used by (other/alive) libraries.
  /// Those parts will not be cleaned up. This is useful when a part has been
  /// "moved" to be part of another library.
  void cleanupSourcesForBuilder(LibraryBuilder builder,
      UriTranslator uriTranslator, Map<Uri, Source> uriToSource,
      [Map<Uri, Source> uriToSourceExtra, Set<Uri> partsUsed]) {
    uriToSource.remove(builder.fileUri);
    uriToSourceExtra?.remove(builder.fileUri);
    Library lib = builder.library;
    for (LibraryPart part in lib.parts) {
      Uri partFileUri = getPartFileUri(lib.fileUri, part, uriTranslator);
      if (partsUsed != null && partsUsed.contains(partFileUri)) continue;
      uriToSource.remove(partFileUri);
      uriToSourceExtra?.remove(partFileUri);
    }
  }

  /// Internal method.
  ///
  /// [partsUsed] indicates part uris that are used by (other/alive) libraries.
  /// Those parts will not be removed from the component problems.
  /// This is useful when a part has been "moved" to be part of another library.
  void removeLibraryFromRemainingComponentProblems(
      Library lib, UriTranslator uriTranslator,
      [Set<Uri> partsUsed]) {
    remainingComponentProblems.remove(lib.fileUri);
    // Remove parts too.
    for (LibraryPart part in lib.parts) {
      Uri partFileUri = getPartFileUri(lib.fileUri, part, uriTranslator);
      remainingComponentProblems.remove(partFileUri);
    }
  }

  /// Internal method.
  int prepareSummary(List<int> summaryBytes, UriTranslator uriTranslator,
      CompilerContext c, IncrementalCompilerData data) {
    dillLoadedData = new DillTarget(ticker, uriTranslator, c.options.target);
    int bytesLength = 0;

    data.component = c.options.target.configureComponent(new Component());
    if (summaryBytes != null) {
      ticker.logMs("Read ${c.options.sdkSummary}");
      new BinaryBuilderWithMetadata(summaryBytes,
              disableLazyReading: false, disableLazyClassReading: true)
          .readComponent(data.component);
      ticker.logMs("Deserialized ${c.options.sdkSummary}");
      bytesLength += summaryBytes.length;
    }

    return bytesLength;
  }

  /// Internal method.
  // This procedure will try to load the dill file and will crash if it cannot.
  Future<int> initializeFromDill(UriTranslator uriTranslator, CompilerContext c,
      IncrementalCompilerData data) async {
    int bytesLength = 0;
    FileSystemEntity entity =
        c.options.fileSystem.entityForUri(initializeFromDillUri);
    if (await entity.exists()) {
      List<int> initializationBytes = await entity.readAsBytes();
      if (initializationBytes != null && initializationBytes.isNotEmpty) {
        ticker.logMs("Read $initializeFromDillUri");
        data.initializationBytes = initializationBytes;

        // We're going to output all we read here so lazy loading it
        // doesn't make sense.
        List<SubComponentView> views = new BinaryBuilderWithMetadata(
                initializationBytes,
                disableLazyReading: true)
            .readComponent(data.component,
                checkCanonicalNames: true, createView: true);

        // Check the any package-urls still point to the same file
        // (e.g. the package still exists and hasn't been updated).
        for (Library lib in data.component.libraries) {
          if (lib.importUri.scheme == "package" &&
              uriTranslator.translate(lib.importUri, false) != lib.fileUri) {
            // Package has been removed or updated.
            // This library should be thrown away.
            // Everything that depends on it should be thrown away.
            // TODO(jensj): Anything that doesn't depend on it can be kept.
            // For now just don't initialize from this dill.
            throw const PackageChangedError();
          }
        }

        // Only initialize the incremental serializer when we know we'll
        // actually use the data loaded from dill.
        initializedIncrementalSerializer =
            incrementalSerializer?.initialize(initializationBytes, views) ??
                false;

        initializedFromDill = true;
        bytesLength += initializationBytes.length;
        saveComponentProblems(data);
      }
    }
    return bytesLength;
  }

  /// Internal method.
  void saveComponentProblems(IncrementalCompilerData data) {
    if (data.component.problemsAsJson != null) {
      for (String jsonString in data.component.problemsAsJson) {
        DiagnosticMessageFromJson message =
            new DiagnosticMessageFromJson.fromJson(jsonString);
        List<DiagnosticMessageFromJson> messages =
            remainingComponentProblems[message.uri] ??=
                new List<DiagnosticMessageFromJson>();
        messages.add(message);
      }
    }
  }

  /// Internal method.
  // This procedure will set up compiler from [componentToInitializeFrom].
  void initializeFromComponent(UriTranslator uriTranslator, CompilerContext c,
      IncrementalCompilerData data) {
    ticker.logMs("About to initializeFromComponent");

    dillLoadedData = new DillTarget(ticker, uriTranslator, c.options.target);
    data.component = new Component(
        libraries: componentToInitializeFrom.libraries,
        uriToSource: componentToInitializeFrom.uriToSource)
      ..setMainMethodAndMode(componentToInitializeFrom.mainMethod?.reference,
          true, componentToInitializeFrom.mode);
    saveComponentProblems(data);

    bool foundDartCore = false;
    for (int i = 0; i < data.component.libraries.length; i++) {
      Library library = data.component.libraries[i];
      if (library.importUri.scheme == "dart" &&
          library.importUri.path == "core") {
        foundDartCore = true;
        break;
      }
    }

    if (!foundDartCore) {
      throw const InitializeFromComponentError("Did not find dart:core when "
          "tried to initialize from component.");
    }

    ticker.logMs("Ran initializeFromComponent");
  }

  /// Internal method.
  void appendLibraries(IncrementalCompilerData data, int bytesLength) {
    if (data.component != null) {
      dillLoadedData.loader
          .appendLibraries(data.component, byteCount: bytesLength);
    }
    ticker.logMs("Appended libraries");
  }

  @override
  Future<Procedure> compileExpression(
      String expression,
      Map<String, DartType> definitions,
      List<TypeParameter> typeDefinitions,
      String syntheticProcedureName,
      Uri libraryUri,
      [String className,
      bool isStatic = false]) async {
    assert(dillLoadedData != null && userCode != null);

    return await context.runInContext((_) async {
      LibraryBuilder libraryBuilder =
          userCode.loader.read(libraryUri, -1, accessor: userCode.loader.first);
      ticker.logMs("Loaded library $libraryUri");

      Class cls;
      if (className != null) {
        ClassBuilder classBuilder = libraryBuilder.scopeBuilder[className];
        cls = classBuilder?.cls;
        if (cls == null) return null;
      }

      userCode.loader.seenMessages.clear();

      for (TypeParameter typeParam in typeDefinitions) {
        if (!isLegalIdentifier(typeParam.name)) return null;
      }
      for (String name in definitions.keys) {
        if (!isLegalIdentifier(name)) return null;
      }

      SourceLibraryBuilder debugLibrary = new SourceLibraryBuilder(
        libraryUri,
        debugExprUri,
        /*packageUri*/ null,
        userCode.loader,
        null,
        scope: libraryBuilder.scope.createNestedScope("expression"),
        nameOrigin: libraryBuilder.library,
      );
      debugLibrary.setLanguageVersion(libraryBuilder.library.languageVersion);
      ticker.logMs("Created debug library");

      if (libraryBuilder is DillLibraryBuilder) {
        for (LibraryDependency dependency
            in libraryBuilder.library.dependencies) {
          if (!dependency.isImport) continue;

          List<Combinator> combinators;

          for (kernel.Combinator combinator in dependency.combinators) {
            combinators ??= <Combinator>[];

            combinators.add(combinator.isShow
                ? new Combinator.show(combinator.names, combinator.fileOffset,
                    libraryBuilder.fileUri)
                : new Combinator.hide(combinator.names, combinator.fileOffset,
                    libraryBuilder.fileUri));
          }

          debugLibrary.addImport(
              null,
              dependency.importedLibraryReference.canonicalName.name,
              null,
              dependency.name,
              combinators,
              dependency.isDeferred,
              -1,
              -1,
              -1,
              -1);
        }

        debugLibrary.addImportsToScope();
        ticker.logMs("Added imports");
      }

      HybridFileSystem hfs = userCode.fileSystem;
      MemoryFileSystem fs = hfs.memory;
      fs.entityForUri(debugExprUri).writeAsStringSync(expression);

      FunctionNode parameters = new FunctionNode(null,
          typeParameters: typeDefinitions,
          positionalParameters: definitions.keys
              .map((name) => new VariableDeclarationImpl(name, 0))
              .toList());

      debugLibrary.build(userCode.loader.coreLibrary, modifyTarget: false);
      Expression compiledExpression = await userCode.loader.buildExpression(
          debugLibrary, className, className != null && !isStatic, parameters);

      Procedure procedure = new Procedure(
          new Name(syntheticProcedureName), ProcedureKind.Method, parameters,
          isStatic: isStatic)
        ..isNonNullableByDefault = debugLibrary.isNonNullableByDefault;

      parameters.body = new ReturnStatement(compiledExpression)
        ..parent = parameters;

      procedure.fileUri = debugLibrary.fileUri;
      procedure.parent = className != null ? cls : libraryBuilder.library;

      userCode.uriToSource.remove(debugExprUri);
      userCode.loader.sourceBytes.remove(debugExprUri);

      // Make sure the library has a canonical name.
      Component c = new Component(libraries: [debugLibrary.library]);
      c.computeCanonicalNames();
      ticker.logMs("Built debug library");

      userCode.runProcedureTransformations(procedure);

      return procedure;
    });
  }

  bool packagesEqual(Package a, Package b) {
    if (a == null || b == null) return false;
    if (a.name != b.name) return false;
    if (a.root != b.root) return false;
    if (a.packageUriRoot != b.packageUriRoot) return false;
    if (a.languageVersion != b.languageVersion) return false;
    if (a.extraData != b.extraData) return false;
    return true;
  }

  /// Internal method.
  ReusageResult computeReusedLibraries(
      Set<Uri> invalidatedUris, UriTranslator uriTranslator) {
    Set<Uri> seenUris = new Set<Uri>();
    List<LibraryBuilder> reusedLibraries = <LibraryBuilder>[];
    for (int i = 0; i < platformBuilders.length; i++) {
      LibraryBuilder builder = platformBuilders[i];
      if (!seenUris.add(builder.importUri)) continue;
      reusedLibraries.add(builder);
    }
    if (userCode == null && userBuilders == null) {
      return new ReusageResult({}, [], false, reusedLibraries);
    }
    bool invalidatedBecauseOfPackageUpdate = false;
    List<LibraryBuilder> directlyInvalidated = new List<LibraryBuilder>();
    Set<LibraryBuilder> notReusedLibraries = new Set<LibraryBuilder>();

    // Maps all non-platform LibraryBuilders from their import URI.
    Map<Uri, LibraryBuilder> builders = <Uri, LibraryBuilder>{};

    // Invalidated URIs translated back to their import URI (package:, dart:,
    // etc.).
    List<Uri> invalidatedImportUris = <Uri>[];

    bool isInvalidated(Uri importUri, Uri fileUri) {
      if (invalidatedUris.contains(importUri)) return true;
      if (importUri != fileUri && invalidatedUris.contains(fileUri)) {
        return true;
      }
      if (hasToCheckPackageUris && importUri.scheme == "package") {
        // Get package name, check if the base URI has changed for the package,
        // if it has, translate the URI again,
        // otherwise the URI cannot have changed.
        String path = importUri.path;
        int firstSlash = path.indexOf('/');
        String packageName = path.substring(0, firstSlash);
        if (previousPackagesMap == null ||
            !packagesEqual(previousPackagesMap[packageName],
                currentPackagesMap[packageName])) {
          Uri newFileUri = uriTranslator.translate(importUri, false);
          if (newFileUri != fileUri) {
            invalidatedBecauseOfPackageUpdate = true;
            return true;
          }
        }
      }
      if (builders[importUri]?.isSynthetic ?? false) return true;
      return false;
    }

    addBuilderAndInvalidateUris(Uri uri, LibraryBuilder libraryBuilder) {
      if (uri.scheme == "dart" && !libraryBuilder.isSynthetic) {
        if (seenUris.add(libraryBuilder.importUri)) {
          reusedLibraries.add(libraryBuilder);
        }
        return;
      }
      builders[uri] = libraryBuilder;
      if (isInvalidated(uri, libraryBuilder.library.fileUri)) {
        invalidatedImportUris.add(uri);
      }
      if (libraryBuilder is SourceLibraryBuilder) {
        for (LibraryBuilder part in libraryBuilder.parts) {
          if (isInvalidated(part.importUri, part.fileUri)) {
            invalidatedImportUris.add(part.importUri);
            builders[part.importUri] = part;
          }
        }
      } else if (libraryBuilder is DillLibraryBuilder) {
        for (LibraryPart part in libraryBuilder.library.parts) {
          Uri partUri = getPartUri(libraryBuilder.importUri, part);
          Uri fileUri = getPartFileUri(
              libraryBuilder.library.fileUri, part, uriTranslator);

          if (isInvalidated(partUri, fileUri)) {
            invalidatedImportUris.add(partUri);
            builders[partUri] = libraryBuilder;
          }
        }
      }
    }

    if (userCode != null) {
      // userCode already contains the builders from userBuilders.
      userCode.loader.builders.forEach(addBuilderAndInvalidateUris);
    } else {
      // userCode was null so we explicitly have to add the builders from
      // userBuilders (which cannot be null as we checked initially that one of
      // them was non-null).
      userBuilders.forEach(addBuilderAndInvalidateUris);
    }

    recordInvalidatedImportUrisForTesting(invalidatedImportUris);
    for (Uri uri in invalidatedImportUris) {
      directlyInvalidated.add(builders[uri]);
    }

    BuilderGraph graph = new BuilderGraph(builders);

    // Compute direct dependencies for each import URI (the reverse of the
    // edges returned by `graph.neighborsOf`).
    Map<Uri, Set<Uri>> directDependencies = <Uri, Set<Uri>>{};
    for (Uri vertex in graph.vertices) {
      for (Uri neighbor in graph.neighborsOf(vertex)) {
        (directDependencies[neighbor] ??= new Set<Uri>()).add(vertex);
      }
    }

    // Remove all dependencies of [invalidatedImportUris] from builders.
    List<Uri> workList = invalidatedImportUris;
    while (workList.isNotEmpty) {
      Uri removed = workList.removeLast();
      LibraryBuilder current = builders.remove(removed);
      // [current] is null if the corresponding key (URI) has already been
      // removed.
      if (current != null) {
        Set<Uri> s = directDependencies[current.importUri];
        if (current.importUri != removed) {
          if (s == null) {
            s = directDependencies[removed];
          } else {
            s.addAll(directDependencies[removed]);
          }
        }
        if (s != null) {
          // [s] is null for leaves.
          for (Uri dependency in s) {
            workList.add(dependency);
          }
        }
        notReusedLibraries.add(current);
      }
    }

    // Builders contain mappings from part uri to builder, meaning the same
    // builder can exist multiple times in the values list.
    for (LibraryBuilder builder in builders.values) {
      if (builder.isPart) continue;
      // TODO(jensj/ahe): This line can probably go away once
      // https://dart-review.googlesource.com/47442 lands.
      if (builder.isPatch) continue;
      if (!seenUris.add(builder.importUri)) continue;
      reusedLibraries.add(builder);
    }

    return new ReusageResult(notReusedLibraries, directlyInvalidated,
        invalidatedBecauseOfPackageUpdate, reusedLibraries);
  }

  @override
  void invalidate(Uri uri) {
    invalidatedUris.add(uri);
  }

  @override
  void invalidateAllSources() {
    if (userCode != null) {
      Set<Uri> uris = new Set<Uri>.from(userCode.loader.builders.keys);
      uris.removeAll(dillLoadedData.loader.builders.keys);
      if (previousSourceBuilders != null) {
        for (Library library in previousSourceBuilders) {
          uris.add(library.importUri);
        }
      }
      invalidatedUris.addAll(uris);
    }
  }

  @override
  void setModulesToLoadOnNextComputeDelta(List<Component> components) {
    modulesToLoad = components.toList();
  }

  /// Internal method.
  void recordNonFullComponentForTesting(Component component) {}

  /// Internal method.
  void recordInvalidatedImportUrisForTesting(List<Uri> uris) {}

  /// Internal method.
  void recordRebuildBodiesCountForTesting(int count) {}

  /// Internal method.
  void recordTemporaryFileForTesting(Uri uri) {}
}

/// Translate a parts "partUri" to an actual uri with handling of invalid uris.
Uri getPartUri(Uri parentUri, LibraryPart part) {
  try {
    return parentUri.resolve(part.partUri);
  } on FormatException {
    // This is also done in [SourceLibraryBuilder.resolve]
    return new Uri(
        scheme: SourceLibraryBuilder.MALFORMED_URI_SCHEME,
        query: Uri.encodeQueryComponent(part.partUri));
  }
}

class PackageChangedError {
  const PackageChangedError();
}

class InitializeFromComponentError {
  final String message;

  const InitializeFromComponentError(this.message);

  String toString() => message;
}

class IncrementalCompilerData {
  Component component = null;
  List<int> initializationBytes = null;
}

class ReusageResult {
  final Set<LibraryBuilder> notReusedLibraries;
  final List<LibraryBuilder> directlyInvalidated;
  final bool invalidatedBecauseOfPackageUpdate;
  final List<LibraryBuilder> reusedLibraries;

  ReusageResult(this.notReusedLibraries, this.directlyInvalidated,
      this.invalidatedBecauseOfPackageUpdate, this.reusedLibraries)
      : assert(notReusedLibraries != null),
        assert(directlyInvalidated != null),
        assert(invalidatedBecauseOfPackageUpdate != null),
        assert(reusedLibraries != null);
}

class ExperimentalInvalidation {
  final Set<LibraryBuilder> rebuildBodies;
  final Set<LibraryBuilder> originalNotReusedLibraries;
  final Set<Uri> missingSources;

  ExperimentalInvalidation(
      this.rebuildBodies, this.originalNotReusedLibraries, this.missingSources)
      : assert(rebuildBodies != null),
        assert(originalNotReusedLibraries != null),
        assert(missingSources != null);
}

class IncrementalKernelTarget extends KernelTarget
    implements ChangedStructureNotifier {
  Set<Class> classHierarchyChanges;
  Set<Class> classMemberChanges;

  IncrementalKernelTarget(FileSystem fileSystem, bool includeComments,
      DillTarget dillTarget, UriTranslator uriTranslator)
      : super(fileSystem, includeComments, dillTarget, uriTranslator);

  ChangedStructureNotifier get changedStructureNotifier => this;

  @override
  void registerClassMemberChange(Class c) {
    classMemberChanges ??= new Set<Class>();
    classMemberChanges.add(c);
  }

  @override
  void registerClassHierarchyChange(Class cls) {
    classHierarchyChanges ??= <Class>{};
    classHierarchyChanges.add(cls);
  }
}