blob: c55f2f98481d5012c5b73b5122f88097e7a2abe3 [file] [log] [blame]
// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
/// Abstraction for a compilation pipeline.
/// A pipeline defines how modular steps are excuted and ensures that a step
/// only has access to the data it declares.
/// The abstract implementation validates how the data is declared, and the
/// underlying implementations enforce the access to data in different ways.
/// The IO-based implementation ensures hermeticity by copying data to different
/// directories. The memory-based implementation ensures hemeticity by filtering
/// out the data before invoking the next step.
import 'suite.dart';
/// Describes a step in a modular compilation pipeline.
class ModularStep {
/// Whether this step needs to read the source files in the module.
final bool needsSources;
/// Data that this step needs to read about dependencies.
/// This can be data produced on a previous stage of the pipeline
/// or produced by this same step when it was run on a dependency.
/// If this list includes any data from [resultData], then the modular-step
/// has to be run on dependencies before it is run on a module. Otherwise, it
/// could be run in parallel.
final List<DataId> dependencyDataNeeded;
/// Data that this step needs to read about the module itself.
/// This is meant to be data produced in earlier stages of the modular
/// pipeline. It is an error to include any id from [resultData] in this list.
final List<DataId> moduleDataNeeded;
/// Data that this step produces.
final List<DataId> resultData;
/// Whether this step is only executed on the main module.
final bool onlyOnMain;
{this.needsSources: true,
this.dependencyDataNeeded: const [],
this.moduleDataNeeded: const [],
this.onlyOnMain: false});
/// Notifies that the step was not executed, but cached instead.
void notifyCached(Module module) {}
/// An object to uniquely identify modular data produced by a modular step.
/// Two modular steps on the same pipeline cannot emit the same data.
class DataId {
final String name;
const DataId(;
String toString() => name;
abstract class Pipeline<S extends ModularStep> {
/// Whether to cache the result of shared modules (e.g. shard packages and sdk
/// libraries) when multiple tests are run by this pipeline.
final bool cacheSharedModules;
final List<S> steps;
Pipeline(this.steps, this.cacheSharedModules) {
void _validate() {
// Ensure that steps consume only data that was produced by previous steps
// or by the same step on a dependency.
Map<DataId, S> previousKinds = {};
for (var step in steps) {
if (step.resultData == null || step.resultData.isEmpty) {
"'${step.runtimeType}' needs to declare what data it produces.");
for (var resultKind in step.resultData) {
if (previousKinds.containsKey(resultKind)) {
_validationError("Cannot produce the same data on two modular steps."
" '$resultKind' was previously produced by "
"'${previousKinds[resultKind].runtimeType}' but "
"'${step.runtimeType}' also produces the same data.");
previousKinds[resultKind] = step;
for (var dataId in step.dependencyDataNeeded) {
if (!previousKinds.containsKey(dataId)) {
"Step '${step.runtimeType}' needs data '${dataId}', but the "
"data is not produced by this or a preceding step.");
for (var dataId in step.moduleDataNeeded) {
if (!previousKinds.containsKey(dataId)) {
"Step '${step.runtimeType}' needs data '${dataId}', but the "
"data is not produced by a preceding step.");
if (dataId == resultKind) {
_validationError("Circular dependency on '$dataId' "
"in step '${step.runtimeType}'");
void _validationError(String s) => throw InvalidPipelineError(s);
Future<void> run(ModularTest test) async {
// TODO(sigmund): validate that [ModularTest] has no cycles.
Map<Module, Set<DataId>> computedData = {};
for (var step in steps) {
await _recursiveRun(step, test.mainModule, computedData, {}, test.flags);
Future<void> _recursiveRun(
S step,
Module module,
Map<Module, Set<DataId>> computedData,
Map<Module, Set<Module>> transitiveDependencies,
List<String> flags) async {
if (transitiveDependencies.containsKey(module)) return;
var deps = transitiveDependencies[module] = {};
for (var dependency in module.dependencies) {
await _recursiveRun(
step, dependency, computedData, transitiveDependencies, flags);
if (step.onlyOnMain && !module.isMain) return;
// Include only requested data from transitive dependencies.
Map<Module, Set<DataId>> visibleData = {};
deps.forEach((dep) {
visibleData[dep] = {};
for (var dataId in step.dependencyDataNeeded) {
if (computedData[dep].contains(dataId)) {
visibleData[module] = {};
for (var dataId in step.moduleDataNeeded) {
if (computedData[module].contains(dataId)) {
await runStep(step, module, visibleData, flags);
(computedData[module] ??= {}).addAll(step.resultData);
Future<void> runStep(S step, Module module,
Map<Module, Set<DataId>> visibleData, List<String> flags);
class InvalidPipelineError extends Error {
final String message;
String toString() => "Invalid pipeline: $message";