Build summary2 in DDC.
New summary2 will unblock fixing long standing inference issues in
Analyzer.
We build it in addition to summary1, and will switch clients to summary2
incrementally. With this CL DDC will build summary2, but will not
consume it yet for analyzing code. This will be done in a separate CL.
This change was also tested in the internal repo, and does not cause
related build failures.
Change-Id: Ic2c635e45b514a2610e938129fa7527fd8d070b4
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/105261
Commit-Queue: Konstantin Shcheglov <scheglov@google.com>
Reviewed-by: Nicholas Shahan <nshahan@google.com>
Reviewed-by: Vijay Menon <vsm@google.com>
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
diff --git a/pkg/analyzer/lib/src/dart/analysis/ddc.dart b/pkg/analyzer/lib/src/dart/analysis/ddc.dart
new file mode 100644
index 0000000..9a37007
--- /dev/null
+++ b/pkg/analyzer/lib/src/dart/analysis/ddc.dart
@@ -0,0 +1,274 @@
+// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:collection';
+
+import 'package:analyzer/dart/analysis/declared_variables.dart';
+import 'package:analyzer/dart/ast/ast.dart';
+import 'package:analyzer/src/dart/analysis/file_state.dart';
+import 'package:analyzer/src/dart/analysis/restricted_analysis_context.dart';
+import 'package:analyzer/src/dart/analysis/session.dart';
+import 'package:analyzer/src/generated/engine.dart';
+import 'package:analyzer/src/generated/source.dart';
+import 'package:analyzer/src/generated/utilities_dart.dart';
+import 'package:analyzer/src/summary/format.dart';
+import 'package:analyzer/src/summary/idl.dart';
+import 'package:analyzer/src/summary/link.dart';
+import 'package:analyzer/src/summary/package_bundle_reader.dart';
+import 'package:analyzer/src/summary/resynthesize.dart';
+import 'package:analyzer/src/summary/summarize_ast.dart';
+import 'package:analyzer/src/summary/summarize_elements.dart';
+import 'package:analyzer/src/summary2/link.dart' as summary2;
+import 'package:analyzer/src/summary2/linked_bundle_context.dart' as summary2;
+import 'package:analyzer/src/summary2/linked_element_factory.dart' as summary2;
+import 'package:analyzer/src/summary2/reference.dart' as summary2;
+import 'package:meta/meta.dart';
+
+class DevCompilerResynthesizerBuilder {
+ final FileSystemState _fsState;
+ final SourceFactory _sourceFactory;
+ final DeclaredVariables _declaredVariables;
+ final AnalysisOptionsImpl _analysisOptions;
+ final SummaryDataStore _summaryData;
+ final List<Uri> _explicitSources;
+
+ _SourceCrawler _fileCrawler;
+
+ final PackageBundleAssembler _assembler;
+ List<int> summaryBytes;
+
+ RestrictedAnalysisContext context;
+ SummaryResynthesizer resynthesizer;
+ summary2.LinkedElementFactory elementFactory;
+
+ DevCompilerResynthesizerBuilder({
+ @required FileSystemState fsState,
+ @required SourceFactory sourceFactory,
+ @required DeclaredVariables declaredVariables,
+ @required AnalysisOptionsImpl analysisOptions,
+ @required SummaryDataStore summaryData,
+ @required List<Uri> explicitSources,
+ }) : _fsState = fsState,
+ _sourceFactory = sourceFactory,
+ _declaredVariables = declaredVariables,
+ _analysisOptions = analysisOptions,
+ _summaryData = summaryData,
+ _explicitSources = explicitSources,
+ _assembler = PackageBundleAssembler();
+
+ /// URIs of libraries that should be linked.
+ List<String> get libraryUris => _fileCrawler.libraryUris;
+
+ /// Link explicit sources, serialize [PackageBundle] into [summaryBytes].
+ ///
+ /// Create a new [context], [resynthesizer] and [elementFactory].
+ void build() {
+ _fileCrawler = _SourceCrawler(
+ _fsState,
+ _sourceFactory,
+ _summaryData,
+ _explicitSources,
+ );
+ _fileCrawler.crawl();
+
+ _buildPackageBundleBytes();
+ var bundle = PackageBundle.fromBuffer(summaryBytes);
+
+ // Create an analysis context to contain the state for this build unit.
+ var synchronousSession = SynchronousSession(
+ _analysisOptions,
+ _declaredVariables,
+ );
+ context = RestrictedAnalysisContext(synchronousSession, _sourceFactory);
+
+ resynthesizer = StoreBasedSummaryResynthesizer(
+ context,
+ null,
+ context.sourceFactory,
+ /*strongMode*/ true,
+ SummaryDataStore([])
+ ..addStore(_summaryData)
+ ..addBundle(null, bundle),
+ );
+ resynthesizer.finishCoreAsyncLibraries();
+ context.typeProvider = resynthesizer.typeProvider;
+ }
+
+ void _buildPackageBundleBytes() {
+ _computeLinkedLibraries1();
+ _computeLinkedLibraries2();
+ summaryBytes = _assembler.assemble().toBuffer();
+ }
+
+ void _computeLinkedLibraries1() {
+ _fileCrawler.sourceToUnlinkedUnit.forEach((source, unlinkedUnit) {
+ _assembler.addUnlinkedUnit(source, unlinkedUnit);
+ });
+
+ var linkResult = link(
+ _fileCrawler.libraryUris.toSet(),
+ (uri) => _summaryData.linkedMap[uri],
+ (uri) =>
+ _summaryData.unlinkedMap[uri] ??
+ _fileCrawler.uriToUnlinkedUnit[uri],
+ _declaredVariables,
+ _analysisOptions);
+ linkResult.forEach(_assembler.addLinkedLibrary);
+ }
+
+ /// Link libraries, and fill [_assembler].
+ void _computeLinkedLibraries2() {
+ var inputLibraries = <summary2.LinkInputLibrary>[];
+
+ var sourceToUnit = _fileCrawler.sourceToUnit;
+ for (var librarySource in sourceToUnit.keys) {
+ var unit = sourceToUnit[librarySource];
+
+ if (_explicitSources.contains(librarySource.uri)) {
+ var isPart = unit.directives.any((d) => d is PartOfDirective);
+ if (isPart) {
+ continue;
+ }
+ }
+
+ var inputUnits = <summary2.LinkInputUnit>[];
+ inputUnits.add(
+ summary2.LinkInputUnit(null, librarySource, false, unit),
+ );
+
+ for (var directive in unit.directives) {
+ if (directive is PartDirective) {
+ var partUri = directive.uri.stringValue;
+ var partSource = _sourceFactory.resolveUri(librarySource, partUri);
+ if (partSource != null) {
+ var partUnit = sourceToUnit[partSource];
+ inputUnits.add(
+ summary2.LinkInputUnit(partUri, partSource, false, partUnit),
+ );
+ }
+ }
+ }
+
+ inputLibraries.add(
+ summary2.LinkInputLibrary(librarySource, inputUnits),
+ );
+ }
+
+ var analysisContext = RestrictedAnalysisContext(
+ SynchronousSession(_analysisOptions, _declaredVariables),
+ _sourceFactory,
+ );
+
+ var elementFactory = summary2.LinkedElementFactory(
+ analysisContext,
+ null,
+ summary2.Reference.root(),
+ );
+
+ for (var bundle in _summaryData.bundles) {
+ elementFactory.addBundle(
+ summary2.LinkedBundleContext(elementFactory, bundle.bundle2),
+ );
+ }
+
+ var linkResult = summary2.link(elementFactory, inputLibraries);
+ _assembler.setBundle2(linkResult.bundle);
+ }
+}
+
+class _SourceCrawler {
+ final FileSystemState _fsState;
+ final SourceFactory _sourceFactory;
+ final SummaryDataStore _summaryData;
+ final List<Uri> _explicitSources;
+
+ /// The pending list of sources to visit.
+ var _pendingSource = Queue<Uri>();
+
+ /// The sources that have been added to [_pendingSource], used to ensure
+ /// we only visit a given source once.
+ var _knownSources = Set<Uri>();
+
+ final Map<Source, UnlinkedUnitBuilder> sourceToUnlinkedUnit = {};
+ final Map<String, UnlinkedUnitBuilder> uriToUnlinkedUnit = {};
+ final Map<Source, CompilationUnit> sourceToUnit = {};
+ final List<String> libraryUris = [];
+
+ _SourceCrawler(
+ this._fsState,
+ this._sourceFactory,
+ this._summaryData,
+ this._explicitSources,
+ );
+
+ /// Starting with [_explicitSources], visit all transitive imports, exports,
+ /// parts, and create an unlinked unit for each (unless it is provided by an
+ /// input summary from [_summaryData]).
+ void crawl() {
+ _pendingSource.addAll(_explicitSources);
+ _knownSources.addAll(_explicitSources);
+
+ // Collect the unlinked units for all transitive sources.
+ //
+ // TODO(jmesserly): consider using parallelism via asynchronous IO here,
+ // once we fix debugger extension (web/web_command.dart) to allow async.
+ //
+ // It would let computation tasks (parsing/serializing unlinked units)
+ // proceed in parallel with reading the sources from disk.
+ while (_pendingSource.isNotEmpty) {
+ _visit(_pendingSource.removeFirst());
+ }
+ }
+
+ /// Visit the file with the given [uri], and fill its data.
+ void _visit(Uri uri) {
+ var uriStr = uri.toString();
+
+ // Maybe an input package contains the source.
+ if (_summaryData.unlinkedMap[uriStr] != null) {
+ return;
+ }
+
+ var source = _sourceFactory.forUri2(uri);
+ if (source == null) {
+ return;
+ }
+
+ var file = _fsState.getFileForPath(source.fullName);
+ var unit = file.parse();
+ sourceToUnit[source] = unit;
+
+ var unlinkedUnit = serializeAstUnlinked(unit);
+ uriToUnlinkedUnit[uriStr] = unlinkedUnit;
+ sourceToUnlinkedUnit[source] = unlinkedUnit;
+
+ void enqueueSource(String relativeUri) {
+ var sourceUri = resolveRelativeUri(uri, Uri.parse(relativeUri));
+ if (_knownSources.add(sourceUri)) {
+ _pendingSource.add(sourceUri);
+ }
+ }
+
+ // Add reachable imports/exports/parts, if any.
+ var isPart = false;
+ for (var directive in unit.directives) {
+ if (directive is UriBasedDirective) {
+ enqueueSource(directive.uri.stringValue);
+ // Handle conditional imports.
+ if (directive is NamespaceDirective) {
+ for (var config in directive.configurations) {
+ enqueueSource(config.uri.stringValue);
+ }
+ }
+ } else if (directive is PartOfDirective) {
+ isPart = true;
+ }
+ }
+
+ // Remember library URIs, for linking and compiling.
+ if (!isPart) {
+ libraryUris.add(uriStr);
+ }
+ }
+}
diff --git a/pkg/dev_compiler/lib/src/analyzer/driver.dart b/pkg/dev_compiler/lib/src/analyzer/driver.dart
index 9e9cb02..24ea8fe 100644
--- a/pkg/dev_compiler/lib/src/analyzer/driver.dart
+++ b/pkg/dev_compiler/lib/src/analyzer/driver.dart
@@ -2,31 +2,24 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
-import 'dart:collection';
import 'dart:typed_data';
import 'package:analyzer/dart/analysis/declared_variables.dart';
-import 'package:analyzer/dart/ast/ast.dart';
import 'package:analyzer/dart/element/element.dart';
import 'package:analyzer/file_system/file_system.dart' show ResourceProvider;
import 'package:analyzer/src/dart/analysis/byte_store.dart';
+import 'package:analyzer/src/dart/analysis/ddc.dart';
import 'package:analyzer/src/dart/analysis/driver.dart' show AnalysisDriver;
import 'package:analyzer/src/dart/analysis/file_state.dart';
import 'package:analyzer/src/dart/analysis/library_analyzer.dart';
import 'package:analyzer/src/dart/analysis/performance_logger.dart';
-import 'package:analyzer/src/dart/analysis/session.dart';
-import 'package:analyzer/src/dart/analysis/restricted_analysis_context.dart';
import 'package:analyzer/src/dart/element/inheritance_manager2.dart';
import 'package:analyzer/src/generated/engine.dart';
import 'package:analyzer/src/generated/resolver.dart' show TypeProvider;
import 'package:analyzer/src/generated/sdk.dart';
import 'package:analyzer/src/generated/source.dart';
-import 'package:analyzer/src/summary/idl.dart';
-import 'package:analyzer/src/summary/link.dart' as summary_link;
import 'package:analyzer/src/summary/package_bundle_reader.dart';
import 'package:analyzer/src/summary/resynthesize.dart';
-import 'package:analyzer/src/summary/summarize_ast.dart';
-import 'package:analyzer/src/summary/summarize_elements.dart';
import 'package:meta/meta.dart';
import '../compiler/shared_command.dart' show sdkLibraryVariables;
@@ -114,26 +107,6 @@
/// [SourceFactory]) and declared variables, if any (`-Dfoo=bar`).
LinkedAnalysisDriver linkLibraries(
List<Uri> explicitSources, AnalyzerOptions options) {
- /// This code was ported from analyzer_cli (with a few changes/improvements).
- ///
- /// Here's a summary of the process:
- ///
- /// 1. starting with [explicitSources], visit all transitive
- /// imports/exports/parts, and create an unlinked unit for each
- /// (unless it's provided by an input summary). Add these to [assembler].
- ///
- /// 2. call [summary_link.link] to create the linked libraries, and add the
- /// results to the assembler.
- ///
- /// 3. serialize the data into [summaryBytes], then deserialize it back into
- /// the [bundle] that contains the summary for all [explicitSources] and
- /// their transitive dependencies.
- ///
- /// 4. create the analysis [context] and element [resynthesizer], and use
- /// them to return a new [LinkedAnalysisDriver] that can analyze all of
- /// the compilation units (and provide the resolved AST/errors for each).
- var assembler = PackageBundleAssembler();
-
/// The URI resolution logic for this build unit.
var sourceFactory = createSourceFactory(options,
sdkResolver: DartUriResolver(dartSdk), summaryData: summaryData);
@@ -141,133 +114,34 @@
/// A fresh file system state for this list of [explicitSources].
var fsState = _createFileSystemState(sourceFactory);
- var uriToUnit = <String, UnlinkedUnit>{};
-
- /// The sources that have been added to [sourcesToProcess], used to ensure
- /// we only visit a given source once.
- var knownSources = HashSet<Uri>.from(explicitSources);
-
- /// The pending list of sources to visit.
- var sourcesToProcess = Queue<Uri>.from(explicitSources);
-
- /// Prepare URIs of unlinked units (for libraries) that should be linked.
- var libraryUris = <String>[];
-
- /// Ensure that the [UnlinkedUnit] for [absoluteUri] is available.
- ///
- /// If the unit is in the input [summaryData], do nothing.
- /// Otherwise compute it and store into the [uriToUnit] and [assembler].
- void prepareUnlinkedUnit(Uri uri) {
- var absoluteUri = uri.toString();
- // Maybe an input package contains the source.
- if (summaryData.unlinkedMap[absoluteUri] != null) {
- return;
- }
- // Parse the source and serialize its AST.
- var source = sourceFactory.forUri2(uri);
- if (source == null || !source.exists()) {
- // Skip this source. We don't need to report an error here because it
- // will be reported later during analysis.
- return;
- }
- var file = fsState.getFileForPath(source.fullName);
- var unit = file.parse();
- var unlinkedUnit = serializeAstUnlinked(unit);
- uriToUnit[absoluteUri] = unlinkedUnit;
- assembler.addUnlinkedUnit(source, unlinkedUnit);
-
- /// The URI to resolve imports/exports/parts against.
- var baseUri = uri;
- if (baseUri.scheme == 'dart' && baseUri.pathSegments.length == 1) {
- // Add a trailing slash so relative URIs will resolve correctly, e.g.
- // "map.dart" from "dart:core/" yields "dart:core/map.dart".
- baseUri = Uri(scheme: 'dart', path: baseUri.path + '/');
- }
-
- void enqueueSource(String relativeUri) {
- var sourceUri = baseUri.resolve(relativeUri);
- if (knownSources.add(sourceUri)) {
- sourcesToProcess.add(sourceUri);
- }
- }
-
- // Add reachable imports/exports/parts, if any.
- var isPart = false;
- for (var directive in unit.directives) {
- if (directive is UriBasedDirective) {
- enqueueSource(directive.uri.stringValue);
- // Handle conditional imports.
- if (directive is NamespaceDirective) {
- for (var config in directive.configurations) {
- enqueueSource(config.uri.stringValue);
- }
- }
- } else if (directive is PartOfDirective) {
- isPart = true;
- }
- }
-
- // Remember library URIs, so we can use it for linking libraries and
- // compiling them.
- if (!isPart) libraryUris.add(absoluteUri);
- }
-
- // Collect the unlinked units for all transitive sources.
- //
- // TODO(jmesserly): consider using parallelism via asynchronous IO here,
- // once we fix debugger extension (web/web_command.dart) to allow async.
- //
- // It would let computation tasks (parsing/serializing unlinked units)
- // proceed in parallel with reading the sources from disk.
- while (sourcesToProcess.isNotEmpty) {
- prepareUnlinkedUnit(sourcesToProcess.removeFirst());
- }
-
var declaredVariables = DeclaredVariables.fromMap(
Map.of(options.declaredVariables)..addAll(sdkLibraryVariables));
- /// Perform the linking step and store the result.
- ///
- /// TODO(jmesserly): can we pass in `getAst` to reuse existing ASTs we
- /// created when we did `file.parse()` in [prepareUnlinkedUnit]?
- var linkResult = summary_link.link(
- libraryUris.toSet(),
- (uri) => summaryData.linkedMap[uri],
- (uri) => summaryData.unlinkedMap[uri] ?? uriToUnit[uri],
- declaredVariables,
- analysisOptions);
- linkResult.forEach(assembler.addLinkedLibrary);
-
- var summaryBytes = assembler.assemble().toBuffer();
- var bundle = PackageBundle.fromBuffer(summaryBytes);
-
- /// Create an analysis context to contain the state for this build unit.
- var synchronousSession =
- SynchronousSession(analysisOptions, declaredVariables);
- var context = RestrictedAnalysisContext(synchronousSession, sourceFactory);
- var resynthesizer = StoreBasedSummaryResynthesizer(
- context,
- null,
- context.sourceFactory,
- /*strongMode*/ true,
- SummaryDataStore([])
- ..addStore(summaryData)
- ..addBundle(null, bundle),
+ var resynthesizerBuilder = DevCompilerResynthesizerBuilder(
+ fsState: fsState,
+ analysisOptions: analysisOptions,
+ declaredVariables: declaredVariables,
+ sourceFactory: sourceFactory,
+ summaryData: summaryData,
+ explicitSources: explicitSources,
);
- resynthesizer.finishCoreAsyncLibraries();
- context.typeProvider = resynthesizer.typeProvider;
+ resynthesizerBuilder.build();
- _extensionTypes ??= ExtensionTypeSet(context.typeProvider, resynthesizer);
+ _extensionTypes ??= ExtensionTypeSet(
+ resynthesizerBuilder.context.typeProvider,
+ resynthesizerBuilder.resynthesizer,
+ );
return LinkedAnalysisDriver(
- analysisOptions,
- resynthesizer,
- sourceFactory,
- libraryUris,
- declaredVariables,
- summaryBytes,
- fsState,
- _resourceProvider);
+ analysisOptions,
+ resynthesizerBuilder.resynthesizer,
+ sourceFactory,
+ resynthesizerBuilder.libraryUris,
+ declaredVariables,
+ resynthesizerBuilder.summaryBytes,
+ fsState,
+ _resourceProvider,
+ );
}
FileSystemState _createFileSystemState(SourceFactory sourceFactory) {
