| #!/usr/bin/env dart | |
| // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file | |
| // for details. All rights reserved. Use of this source code is governed by a | |
| // BSD-style license that can be found in the LICENSE file. | |
| /// Command line tool to merge the SDK libraries and our patch files. | |
| /// This is currently designed as an offline tool, but we could automate it. | |
| import 'dart:io'; | |
| import 'dart:math' as math; | |
| import 'package:analyzer/analyzer.dart'; | |
| import 'package:analyzer/dart/ast/token.dart'; | |
| import 'package:analyzer/src/generated/sdk.dart'; | |
| import 'package:path/path.dart' as path; | |
| void main(List<String> argv) { | |
| var self = path.relative(path.fromUri(Platform.script)); | |
| if (argv.length < 3) { | |
| var toolDir = path.relative(path.dirname(path.fromUri(Platform.script))); | |
| var repoExample = path.join(toolDir, '..', '..', '..'); | |
| var patchExample = path.join(toolDir, 'input_sdk'); | |
| var outExample = | |
| path.relative(path.normalize(path.join('gen', 'patched_sdk'))); | |
| print('Usage: $self DART_REPO_DIR PATCH_DIR OUTPUT_DIR'); | |
| print('For example:'); | |
| print('\$ $self $repoExample $patchExample $outExample'); | |
| exit(1); | |
| } | |
| var selfModifyTime = File(self).lastModifiedSync().millisecondsSinceEpoch; | |
| var repoDir = argv[0]; | |
| var patchDir = argv[1]; | |
| var sdkLibIn = path.join(repoDir, 'sdk', 'lib'); | |
| var patchIn = path.join(patchDir, 'patch'); | |
| var privateIn = path.join(patchDir, 'private'); | |
| var sdkOut = path.join(argv[2], 'lib'); | |
| var INTERNAL_PATH = '_internal/js_runtime/lib/'; | |
| // Copy libraries.dart and version | |
| var librariesDart = path.join(patchDir, 'libraries.dart'); | |
| var libContents = File(librariesDart).readAsStringSync(); | |
| // TODO(jmesserly): can we remove this? | |
| _writeSync(path.join(sdkOut, '_internal', 'libraries.dart'), libContents); | |
| _writeSync( | |
| path.join( | |
| sdkOut, '_internal', 'sdk_library_metadata', 'lib', 'libraries.dart'), | |
| libContents); | |
| _writeSync(path.join(sdkOut, '..', 'version'), | |
| File(path.join(repoDir, 'tools', 'VERSION')).readAsStringSync()); | |
| // Parse libraries.dart | |
| var sdkLibraries = _getSdkLibraries(libContents); | |
| // Enumerate core libraries and apply patches | |
| for (SdkLibrary library in sdkLibraries) { | |
| // TODO(jmesserly): analyzer does not handle the default case of | |
| // "both platforms" correctly, and treats it as being supported on neither. | |
| // So instead we skip explicitly marked as VM libs. | |
| if (library.isVmLibrary) continue; | |
| var libraryOut = path.join(sdkLibIn, library.path); | |
| var libraryOverride = path.join(patchDir, 'lib', library.path); | |
| String libraryIn; | |
| if (library.path.contains(INTERNAL_PATH)) { | |
| libraryIn = | |
| path.join(privateIn, library.path.replaceAll(INTERNAL_PATH, '')); | |
| } else if (File(libraryOverride).existsSync()) { | |
| libraryIn = libraryOverride; | |
| } else { | |
| libraryIn = libraryOut; | |
| } | |
| var libraryFile = File(libraryIn); | |
| if (libraryFile.existsSync()) { | |
| var outPaths = <String>[libraryOut]; | |
| var libraryContents = libraryFile.readAsStringSync(); | |
| int inputModifyTime = math.max(selfModifyTime, | |
| libraryFile.lastModifiedSync().millisecondsSinceEpoch); | |
| var partFiles = <File>[]; | |
| for (var part in parseDirectives(libraryContents).directives) { | |
| if (part is PartDirective) { | |
| var partPath = part.uri.stringValue; | |
| outPaths.add(path.join(path.dirname(libraryOut), partPath)); | |
| var partFile = File(path.join(path.dirname(libraryIn), partPath)); | |
| partFiles.add(partFile); | |
| inputModifyTime = math.max(inputModifyTime, | |
| partFile.lastModifiedSync().millisecondsSinceEpoch); | |
| } | |
| } | |
| // See if we can find a patch file. | |
| var patchPath = path.join( | |
| patchIn, path.basenameWithoutExtension(libraryIn) + '_patch.dart'); | |
| var patchFile = File(patchPath); | |
| bool patchExists = patchFile.existsSync(); | |
| if (patchExists) { | |
| inputModifyTime = math.max(inputModifyTime, | |
| patchFile.lastModifiedSync().millisecondsSinceEpoch); | |
| } | |
| // Compute output paths | |
| outPaths = outPaths | |
| .map((p) => path.join(sdkOut, path.relative(p, from: sdkLibIn))) | |
| .toList(); | |
| // Compare output modify time with input modify time. | |
| bool needsUpdate = false; | |
| for (var outPath in outPaths) { | |
| var outFile = File(outPath); | |
| if (!outFile.existsSync() || | |
| outFile.lastModifiedSync().millisecondsSinceEpoch < | |
| inputModifyTime) { | |
| needsUpdate = true; | |
| break; | |
| } | |
| } | |
| if (needsUpdate) { | |
| var contents = <String>[libraryContents]; | |
| contents.addAll(partFiles.map((f) => f.readAsStringSync())); | |
| if (patchExists) { | |
| var patchContents = patchFile.readAsStringSync(); | |
| contents = _patchLibrary(contents, patchContents); | |
| } | |
| if (contents != null) { | |
| for (var i = 0; i < outPaths.length; i++) { | |
| _writeSync(outPaths[i], contents[i]); | |
| } | |
| } else { | |
| exitCode = 2; | |
| } | |
| } | |
| } | |
| } | |
| } | |
| /// Writes a file, creating the directory if needed. | |
| void _writeSync(String filePath, String contents) { | |
| var outDir = Directory(path.dirname(filePath)); | |
| if (!outDir.existsSync()) outDir.createSync(recursive: true); | |
| File(filePath).writeAsStringSync(contents); | |
| } | |
| /// Merges dart:* library code with code from *_patch.dart file. | |
| /// | |
| /// Takes a list of the library's parts contents, with the main library contents | |
| /// first in the list, and the contents of the patch file. | |
| /// | |
| /// The result will have `@patch` implementations merged into the correct place | |
| /// (e.g. the class or top-level function declaration) and all other | |
| /// declarations introduced by the patch will be placed into the main library | |
| /// file. | |
| /// | |
| /// This is purely a syntactic transformation. Unlike dart2js patch files, there | |
| /// is no semantic meaning given to the *_patch files, and they do not magically | |
| /// get their own library scope, etc. | |
| /// | |
| /// Editorializing: the dart2js approach requires a Dart front end such as | |
| /// package:analyzer to semantically model a feature beyond what is specified | |
| /// in the Dart language. Since this feature is only for the convenience of | |
| /// writing the dart:* libraries, and not a tool given to Dart developers, it | |
| /// seems like a non-ideal situation. Instead we keep the preprocessing simple. | |
| List<String> _patchLibrary(List<String> partsContents, String patchContents) { | |
| var results = <StringEditBuffer>[]; | |
| // Parse the patch first. We'll need to extract bits of this as we go through | |
| // the other files. | |
| var patchFinder = PatchFinder.parseAndVisit(patchContents); | |
| // Merge `external` declarations with the corresponding `@patch` code. | |
| bool failed = false; | |
| for (var partContent in partsContents) { | |
| var partEdits = StringEditBuffer(partContent); | |
| var partUnit = parseCompilationUnit(partContent); | |
| var patcher = PatchApplier(partEdits, patchFinder); | |
| partUnit.accept(patcher); | |
| if (!failed) failed = patcher.patchWasMissing; | |
| results.add(partEdits); | |
| } | |
| if (failed) return null; | |
| return List<String>.from(results.map((e) => e.toString())); | |
| } | |
| /// Merge `@patch` declarations into `external` declarations. | |
| class PatchApplier extends GeneralizingAstVisitor { | |
| final StringEditBuffer edits; | |
| final PatchFinder patch; | |
| bool _isLibrary = true; // until proven otherwise. | |
| bool patchWasMissing = false; | |
| PatchApplier(this.edits, this.patch); | |
| @override | |
| visitCompilationUnit(CompilationUnit node) { | |
| super.visitCompilationUnit(node); | |
| if (_isLibrary) _mergeUnpatched(node); | |
| } | |
| void _merge(AstNode node, int pos) { | |
| var code = patch.contents.substring(node.offset, node.end); | |
| edits.insert(pos, '\n' + code); | |
| } | |
| /// Merges directives and declarations that are not `@patch` into the library. | |
| void _mergeUnpatched(CompilationUnit unit) { | |
| // Merge imports from the patch | |
| // TODO(jmesserly): remove duplicate imports | |
| // To patch a library, we must have a library directive | |
| var libDir = unit.directives.first as LibraryDirective; | |
| int importPos = unit.directives | |
| .lastWhere((d) => d is ImportDirective, orElse: () => libDir) | |
| .end; | |
| for (var d in patch.unit.directives.where((d) => d is ImportDirective)) { | |
| _merge(d, importPos); | |
| } | |
| int partPos = unit.directives.last.end; | |
| for (var d in patch.unit.directives.where((d) => d is PartDirective)) { | |
| _merge(d, partPos); | |
| } | |
| // Merge declarations from the patch | |
| int declPos = edits.original.length; | |
| for (var d in patch.mergeDeclarations) { | |
| _merge(d, declPos); | |
| } | |
| } | |
| @override | |
| visitPartOfDirective(PartOfDirective node) { | |
| _isLibrary = false; | |
| } | |
| @override | |
| visitFunctionDeclaration(FunctionDeclaration node) { | |
| _maybePatch(node); | |
| } | |
| /// Merge patches and extensions into the class | |
| @override | |
| visitClassDeclaration(ClassDeclaration node) { | |
| node.members.forEach(_maybePatch); | |
| var mergeMembers = patch.mergeMembers[_qualifiedName(node)]; | |
| if (mergeMembers == null) return; | |
| // Merge members from the patch | |
| var pos = node.members.last.end; | |
| for (var member in mergeMembers) { | |
| var code = patch.contents.substring(member.offset, member.end); | |
| edits.insert(pos, '\n\n ' + code); | |
| } | |
| } | |
| void _maybePatch(Declaration node) { | |
| if (node is FieldDeclaration) return; | |
| Token externalKeyword = (node as dynamic).externalKeyword; | |
| if (externalKeyword == null) return; | |
| var name = _qualifiedName(node); | |
| var patchNode = patch.patches[name]; | |
| if (patchNode == null) { | |
| print('warning: patch not found for $name: $node'); | |
| patchWasMissing = true; | |
| return; | |
| } | |
| Annotation patchMeta = patchNode.metadata.lastWhere(_isPatchAnnotation); | |
| int start = patchMeta.endToken.next.offset; | |
| var code = patch.contents.substring(start, patchNode.end); | |
| // Const factory constructors can't be legally parsed from the patch file, | |
| // so we need to omit the "const" there, but still preserve it. | |
| if (node is ConstructorDeclaration && | |
| node.constKeyword != null && | |
| patchNode is ConstructorDeclaration && | |
| patchNode.constKeyword == null) { | |
| code = 'const $code'; | |
| } | |
| // For some node like static fields, the node's offset doesn't include | |
| // the external keyword. Also starting from the keyword lets us preserve | |
| // documentation comments. | |
| edits.replace(externalKeyword.offset, node.end, code); | |
| } | |
| } | |
| class PatchFinder extends GeneralizingAstVisitor { | |
| final String contents; | |
| final CompilationUnit unit; | |
| final patches = <String, Declaration>{}; | |
| final mergeMembers = <String, List<ClassMember>>{}; | |
| final mergeDeclarations = <CompilationUnitMember>[]; | |
| PatchFinder.parseAndVisit(String contents) | |
| : contents = contents, | |
| unit = parseCompilationUnit(contents) { | |
| visitCompilationUnit(unit); | |
| } | |
| @override | |
| visitCompilationUnitMember(CompilationUnitMember node) { | |
| mergeDeclarations.add(node); | |
| } | |
| @override | |
| visitClassDeclaration(ClassDeclaration node) { | |
| if (_isPatch(node)) { | |
| var members = <ClassMember>[]; | |
| for (var member in node.members) { | |
| if (_isPatch(member)) { | |
| patches[_qualifiedName(member)] = member; | |
| } else { | |
| members.add(member); | |
| } | |
| } | |
| if (members.isNotEmpty) { | |
| mergeMembers[_qualifiedName(node)] = members; | |
| } | |
| } else { | |
| mergeDeclarations.add(node); | |
| } | |
| } | |
| @override | |
| visitFunctionDeclaration(FunctionDeclaration node) { | |
| if (_isPatch(node)) { | |
| patches[_qualifiedName(node)] = node; | |
| } else { | |
| mergeDeclarations.add(node); | |
| } | |
| } | |
| @override | |
| visitFunctionBody(node) {} // skip method bodies | |
| } | |
| String _qualifiedName(Declaration node) { | |
| var result = ""; | |
| var parent = node.parent; | |
| if (parent is ClassDeclaration) { | |
| result = "${parent.name.name}."; | |
| } | |
| SimpleIdentifier name = (node as dynamic).name; | |
| if (name != null) result += name.name; | |
| // Make sure setters and getters don't collide. | |
| if (node is FunctionDeclaration && node.isSetter || | |
| node is MethodDeclaration && node.isSetter) { | |
| result += "="; | |
| } | |
| return result; | |
| } | |
| bool _isPatch(AnnotatedNode node) => node.metadata.any(_isPatchAnnotation); | |
| bool _isPatchAnnotation(Annotation m) => | |
| m.name.name == 'patch' && m.constructorName == null && m.arguments == null; | |
| /// Editable string buffer. | |
| /// | |
| /// Applies a series of edits (insertions, removals, replacements) using | |
| /// original location information, and composes them into the edited string. | |
| /// | |
| /// For example, starting with a parsed AST with original source locations, | |
| /// this type allows edits to be made without regards to other edits. | |
| class StringEditBuffer { | |
| final String original; | |
| final _edits = <_StringEdit>[]; | |
| /// Creates a new transaction. | |
| StringEditBuffer(this.original); | |
| bool get hasEdits => _edits.length > 0; | |
| /// Edit the original text, replacing text on the range [begin] and | |
| /// exclusive [end] with the [replacement] string. | |
| void replace(int begin, int end, String replacement) { | |
| _edits.add(_StringEdit(begin, end, replacement)); | |
| } | |
| /// Insert [string] at [offset]. | |
| /// Equivalent to `replace(offset, offset, string)`. | |
| void insert(int offset, String string) => replace(offset, offset, string); | |
| /// Remove text from the range [begin] to exclusive [end]. | |
| /// Equivalent to `replace(begin, end, '')`. | |
| void remove(int begin, int end) => replace(begin, end, ''); | |
| /// Applies all pending [edit]s and returns a new string. | |
| /// | |
| /// This method is non-destructive: it does not discard existing edits or | |
| /// change the [original] string. Further edits can be added and this method | |
| /// can be called again. | |
| /// | |
| /// Throws [UnsupportedError] if the edits were overlapping. If no edits were | |
| /// made, the original string will be returned. | |
| String toString() { | |
| var sb = StringBuffer(); | |
| if (_edits.length == 0) return original; | |
| // Sort edits by start location. | |
| _edits.sort(); | |
| int consumed = 0; | |
| for (var edit in _edits) { | |
| if (consumed > edit.begin) { | |
| sb = StringBuffer(); | |
| sb.write('overlapping edits. Insert at offset '); | |
| sb.write(edit.begin); | |
| sb.write(' but have consumed '); | |
| sb.write(consumed); | |
| sb.write(' input characters. List of edits:'); | |
| for (var e in _edits) { | |
| sb.write('\n '); | |
| sb.write(e); | |
| } | |
| throw UnsupportedError(sb.toString()); | |
| } | |
| // Add characters from the original string between this edit and the last | |
| // one, if any. | |
| var betweenEdits = original.substring(consumed, edit.begin); | |
| sb.write(betweenEdits); | |
| sb.write(edit.replace); | |
| consumed = edit.end; | |
| } | |
| // Add any text from the end of the original string that was not replaced. | |
| sb.write(original.substring(consumed)); | |
| return sb.toString(); | |
| } | |
| } | |
| class _StringEdit implements Comparable<_StringEdit> { | |
| final int begin; | |
| final int end; | |
| final String replace; | |
| _StringEdit(this.begin, this.end, this.replace); | |
| int get length => end - begin; | |
| String toString() => '(Edit @ $begin,$end: "$replace")'; | |
| int compareTo(_StringEdit other) { | |
| int diff = begin - other.begin; | |
| if (diff != 0) return diff; | |
| return end - other.end; | |
| } | |
| } | |
| List<SdkLibrary> _getSdkLibraries(String contents) { | |
| // TODO(jmesserly): fix SdkLibrariesReader_LibraryBuilder in Analyzer. | |
| // It doesn't understand optional new/const in Dart 2. For now, we keep | |
| // redundant `const` in tool/input_sdk/libraries.dart as a workaround. | |
| var libraryBuilder = SdkLibrariesReader_LibraryBuilder(true); | |
| parseCompilationUnit(contents).accept(libraryBuilder); | |
| return libraryBuilder.librariesMap.sdkLibraries; | |
| } |