updates to the deps tool (#61)
diff --git a/pkgs/corpus/README.md b/pkgs/corpus/README.md
index 73ee42a..7973970 100644
--- a/pkgs/corpus/README.md
+++ b/pkgs/corpus/README.md
@@ -1,38 +1,44 @@
+Welcome! There are two tools in this repo - a `deps` tool and a `usage` tool. To
+use them, clone this repo, cd to `pkgs/corpus`, and run one of the two tools
+below.
-## What's this?
+## bin/deps.dart
+
+This is a tool to calculate information about which packages depend on a target
+package.
+
+It queries pub.dev for the packages that use `<package-name>` and generates a
+`.csv` file with the dependent information. This is useful for things like
+understanding which packages might be impacted by version changes to the target
+package. For an example of the dependency report, see
+[matcher.csv](doc/matcher.csv).
+
+### Usage
+
+```
+usage: dart bin/deps.dart [options] <package-name>
+
+options:
+-h, --help Print this usage information.
+ --package-limit=<count> Limit the number of packages to return data for.
+ --include-old Include packages that haven't been published in the last year.
+ --include-dev-deps Include usages from dev dependencies.
+```
+
+## bin/usage.dart
This is a tool to calculate the API usage for a package - what parts of a
package's API are typically used by other Dart packages and applications.
-It is run from the command line:
-
-```
-dart bin/api_usage.dart <package-name>
-```
-
It queries pub.dev for the packages that use `<package-name>`, downloads the
referencing packages, analyzes them, and determines which portions of the target
package's API are used. For an example usage report, see
[collection.md](doc/collection.md).
-## Usage
-
-To use this tool, clone the repo, cd to `packages/corpus`, and run:
+### Usage
```
-dart bin/api_usage.dart <package-name>
-```
-
-Some available options are:
-
-- `--package-limit`: limit the number of packages that are used for analysis
-- `--show-src-references`: when there are references into a package's `lib/src/`
- directory (something that's generally not intended to be part of a package's
- public API), this option will include which package is using the `src/`
- library in the output
-
-```
-usage: dart bin/api_usage.dart [options] <package-name>
+usage: dart bin/usage.dart [options] <package-name>
options:
-h, --help Print this usage information.
diff --git a/pkgs/corpus/bin/deps.dart b/pkgs/corpus/bin/deps.dart
index dd17a35..be25c2d 100644
--- a/pkgs/corpus/bin/deps.dart
+++ b/pkgs/corpus/bin/deps.dart
@@ -2,8 +2,7 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
-/// Outputs information in CSV format for all the dependencies of a given
-/// package.
+/// Outputs information in CSV format for all the dependents of a given package.
import 'dart:io';
@@ -11,8 +10,16 @@
import 'package:cli_util/cli_logging.dart';
import 'package:corpus/packages.dart';
import 'package:corpus/pub.dart';
+import 'package:path/path.dart' as path;
+import 'package:pub_semver/pub_semver.dart';
-// TODO: show the 'discontinued' property
+const packageLimitFlag = 'package-limit';
+const includeOldFlag = 'include-old';
+const includeDevDepsFlag = 'include-dev-deps';
+const requireSdk212 = true;
+
+// TODO: Turn this and 'usage.dart' into a combined CommandRunner tool.
+// TODO: Move the bulk of the implementation into lib/ (to facilitate testing).
void main(List<String> args) async {
var argParser = createArgParser();
@@ -33,8 +40,9 @@
}
final packageName = argResults.rest.first;
- var packageLimit = argResults['package-limit'] as String?;
- var excludeOld = argResults['exclude-old'] as bool;
+ var packageLimit = argResults[packageLimitFlag] as String?;
+ var includeOld = argResults[includeOldFlag] as bool;
+ var includeDevDeps = argResults[includeDevDepsFlag] as bool;
var log = Logger.standard();
@@ -52,10 +60,7 @@
var limit = packageLimit == null ? null : int.parse(packageLimit);
- var packages = await pub.dependenciesOf(
- packageName,
- limit: limit == null ? null : limit * 2,
- );
+ var packageStream = pub.popularDependenciesOf(packageName);
progress.finish(showTiming: true);
@@ -63,17 +68,34 @@
var count = 0;
- for (var package in packages) {
- progress = log.progress(' $package');
+ await for (var package in packageStream) {
var usage = await getPackageUsageInfo(pub, package);
- progress.finish();
- if (excludeOld) {
- if (!usage.packageInfo.publishedDate.isAfter(dateOneYearAgo)) {
+ if (usage.packageOptions.isDiscontinued) {
+ continue;
+ }
+
+ if (!includeOld &&
+ !usage.packageInfo.publishedDate.isAfter(dateOneYearAgo)) {
+ continue;
+ }
+
+ var constraintType = package.constraintType(targetPackage.name);
+ if (!includeDevDeps && constraintType == 'dev') {
+ continue;
+ }
+
+ var sdkConstraint = usage.packageInfo.sdkConstraint;
+ if (requireSdk212 && sdkConstraint != null) {
+ // We only want packages that support 2.12 and later. As a close proxy, we
+ // skip packages that allow 2.11.0.
+ var constraint = VersionConstraint.parse(sdkConstraint);
+ if (constraint.allows(Version.parse('2.11.0'))) {
continue;
}
}
+ log.stdout(' $package');
usageInfos.add(usage);
count++;
@@ -101,12 +123,12 @@
PackageUsageInfo(this.packageInfo, this.packageOptions, this.packageScore);
}
-Future<PackageUsageInfo> getPackageUsageInfo(Pub pub, String package) async {
- var packageInfo = await pub.getPackageInfo(package);
- var packageOptions = await pub.getPackageOptions(packageInfo.name);
- var packageScore = await pub.getPackageScore(packageInfo.name);
+Future<PackageUsageInfo> getPackageUsageInfo(
+ Pub pub, PackageInfo package) async {
+ var packageOptions = await pub.getPackageOptions(package.name);
+ var packageScore = await pub.getPackageScore(package.name);
- return PackageUsageInfo(packageInfo, packageOptions, packageScore);
+ return PackageUsageInfo(package, packageOptions, packageScore);
}
File generateCsvReport(
@@ -115,29 +137,39 @@
) {
var buf = StringBuffer();
+ var columns = [
+ Column('Package', (usage) => usage.packageInfo.name),
+ Column('Version', (usage) => usage.packageInfo.version),
+ Column('Publish Days', (usage) => daysOld(usage.packageInfo.publishedDate)),
+ Column('Score', (usage) {
+ var score = usage.packageScore;
+ return printDouble(score.grantedPoints * 100 / score.maxPoints);
+ }),
+ Column('Popularity',
+ (usage) => printDouble(usage.packageScore.popularityScore * 100)),
+ Column('Likes', (usage) => '${usage.packageScore.likeCount}'),
+ Column('Constraint',
+ (usage) => '${usage.packageInfo.constraintFor(targetPackage.name)}'),
+ Column('Dep Type',
+ (usage) => '${usage.packageInfo.constraintType(targetPackage.name)}'),
+ Column('SDK', (usage) => '${usage.packageInfo.sdkConstraint}'),
+ Column('Repo', (usage) => '${usage.packageInfo.repo}'),
+ ];
+
buf.writeln('${targetPackage.name} ${targetPackage.version}');
buf.writeln();
- buf.writeln('Package,Version,Last Published (days),Popularity,Quality Score,'
- 'SDK Constraint,Package Constraint,Constraint Type,Likes,Repo');
+ buf.writeln(columns.map((c) => c.title).join(','));
+
+ // Sort the packages by the number of likes.
+ usageInfos.sort((usage1, usage2) {
+ return usage2.packageScore.likeCount - usage1.packageScore.likeCount;
+ });
for (var usage in usageInfos) {
- var package = usage.packageInfo;
- var score = usage.packageScore;
- buf.writeln(
- '${package.name},'
- '${package.version},'
- '${daysOld(package.publishedDate)},'
- '${printDouble(score.popularityScore * 100)},'
- '${printDouble(score.grantedPoints * 100 / score.maxPoints)},'
- '${package.sdkConstraint ?? ''},'
- '${package.constraintFor(targetPackage.name) ?? ''},'
- '${package.constraintType(targetPackage.name) ?? ''},'
- '${score.likeCount},'
- '${package.repo ?? ''}',
- );
+ buf.writeln(columns.map((c) => c.fn(usage)).join(','));
}
- var file = File('reports/${targetPackage.name}.csv');
+ var file = File(path.join('reports', '${targetPackage.name}.csv'));
file.parent.createSync();
file.writeAsStringSync(buf.toString());
return file;
@@ -152,14 +184,19 @@
help: 'Print this usage information.',
);
parser.addOption(
- 'package-limit',
- help: 'Limit the number of packages usage data is collected from.',
+ packageLimitFlag,
+ help: 'Limit the number of packages to return data for.',
valueHelp: 'count',
);
parser.addFlag(
- 'exclude-old',
+ includeOldFlag,
negatable: false,
- help: 'Exclude packages that haven\'t been published in the last year.',
+ help: "Include packages that haven't been published in the last year.",
+ );
+ parser.addFlag(
+ includeDevDepsFlag,
+ negatable: false,
+ help: 'Include usages from dev dependencies.',
);
return parser;
}
@@ -179,3 +216,10 @@
}
String printDouble(double value) => '${value.round()}';
+
+class Column {
+ final String title;
+ final String Function(PackageUsageInfo) fn;
+
+ Column(this.title, this.fn);
+}
diff --git a/pkgs/corpus/bin/api_usage.dart b/pkgs/corpus/bin/usage.dart
similarity index 93%
rename from pkgs/corpus/bin/api_usage.dart
rename to pkgs/corpus/bin/usage.dart
index 980413e..3d6416b 100644
--- a/pkgs/corpus/bin/api_usage.dart
+++ b/pkgs/corpus/bin/usage.dart
@@ -5,7 +5,7 @@
import 'dart:io';
import 'package:args/args.dart';
-import 'package:corpus/api_usage.dart';
+import 'package:corpus/usage.dart';
void main(List<String> args) async {
var argParser = _createArgParser();
@@ -60,7 +60,7 @@
}
void _printUsage(ArgParser argParser) {
- print('usage: dart bin/api_usage.dart [options] <package-name>');
+ print('usage: dart bin/usage.dart [options] <package-name>');
print('');
print('options:');
print(argParser.usage);
diff --git a/pkgs/corpus/doc/matcher.csv b/pkgs/corpus/doc/matcher.csv
new file mode 100644
index 0000000..8701ad5
--- /dev/null
+++ b/pkgs/corpus/doc/matcher.csv
@@ -0,0 +1,34 @@
+matcher 0.12.14
+
+Package,Version,Publish Days,Score,Popularity,Likes,Constraint,Dep Type,SDK,Repo
+mockito,5.3.2,140,93,99,930,^0.12.10,regular,>=2.17.0-0 <3.0.0,https://github.com/dart-lang/mockito
+mocktail,0.3.0,344,100,98,698,^0.12.10,regular,>=2.12.0 <3.0.0,https://github.com/felangel/mocktail
+quiver,3.2.1,50,93,99,434,^0.12.10,regular,>=2.17.0 <3.0.0,https://github.com/google/quiver-dart
+dcli,2.0.1,5,93,91,166,^0.12.14,regular,>=2.19.0 <3.0.0,https://github.com/onepub-dev/dcli
+code_builder,4.4.0,50,100,96,154,^0.12.10,regular,>=2.17.0 <3.0.0,https://github.com/dart-lang/code_builder
+mockingjay,0.3.0,310,100,93,67,^0.12.10,regular,>=2.12.0 <3.0.0,https://github.com/VeryGoodOpenSource/mockingjay
+leak_tracker,2.0.1,13,93,88,54,^0.12.13,regular,>=2.18.0 <3.0.0,https://github.com/dart-lang/leak_tracker
+webdriver,3.0.2,22,79,100,35,^0.12.10,regular,>=2.18.0 <3.0.0,https://github.com/google/webdriver.dart
+gherkin,3.1.0,223,93,86,33,^0.12.11,regular,>=2.15.0 <3.0.0,https://github.com/jonsamwell/dart_gherkin
+angel3_framework,7.0.3,126,93,77,33,^0.12.10,regular,>=2.17.0 <3.0.0,https://github.com/dukefirehawk/angel/tree/master/packages/framework
+test_api,0.4.18,28,86,99,10,>=0.12.11 <0.12.15,regular,>=2.18.0 <3.0.0,https://github.com/dart-lang/test/tree/master/pkgs/test_api
+rx,0.2.0,4,100,78,8,^0.12.0,regular,>=2.17.0 <3.0.0,https://github.com/renggli/dart-rx
+build_test,2.1.6,20,86,93,8,^0.12.0,regular,>=2.18.0 <3.0.0,https://github.com/dart-lang/build/tree/master/build_test
+typed_result,1.0.0,21,100,54,4,^0.12.14,regular,>=2.18.2 <3.0.0,https://github.com/lucastsantos/typed_result
+flutter_lwa,2.1.0,46,93,82,3,^0.12.10,regular,>=2.12.0 <3.0.0,https://github.com/ayvazj/flutter_lwa
+built_value_test,8.4.3,29,93,58,3,^0.12.0,regular,>=2.12.0 <3.0.0,https://github.com/google/built_value.dart/tree/master/built_value_test
+angel3_validate,7.0.1,68,100,71,2,^0.12.0,regular,>=2.17.0 <3.0.0,https://github.com/dukefirehawk/angel/tree/master/packages/validate
+conduit_test,4.2.2,8,100,70,2,^0.12.12,regular,>=2.19.0 <3.0.0,https://github.com/conduit-dart/conduit
+test_descriptor,2.0.1,113,79,83,2,^0.12.10,regular,>=2.12.0 <3.0.0,https://github.com/dart-lang/test_descriptor
+ngpageloader,5.0.0,184,86,6,2,^0.12.10,regular,>=2.17.0 <3.0.0,https://github.com/angulardart-community/ngpageloader
+test_core,0.4.23,1,93,100,1,any,regular,>=2.18.0 <3.0.0,https://github.com/dart-lang/test/tree/master/pkgs/test_core
+jael3,7.0.0,165,93,57,1,^0.12.10,regular,>=2.17.0 <3.0.0,https://github.com/dukefirehawk/angel/tree/master/packages/jael/jael
+dartbag,0.6.0,8,93,31,1,^0.12.14,regular,>=2.19.0 <3.0.0,https://github.com/jamesderlin/dartbag
+critical_test,7.0.1,85,64,0,1,0.12.13,regular,>=2.18.0 <3.0.0,https://github.com/bsutton/critical_tester
+dev_test,0.15.7+1,14,100,79,0,>=0.12.10 <2.0.0,regular,>=2.18.0 <3.0.0,https://github.com/tekartik/dev_test.dart/tree/master/dev_test
+mock_exceptions,0.8.2,37,100,69,0,^0.12.12,regular,>=2.18.6 <3.0.0,https://github.com/atn832/mock_exceptions
+belatuk_combinator,4.0.0,218,100,49,0,^0.12.10,regular,>=2.17.0 <3.0.0,https://github.com/dart-backend/belatuk-common-utilities/tree/main/packages/combinator
+angel3_test,7.0.0,166,100,39,0,^0.12.10,regular,>=2.17.0 <3.0.0,https://github.com/dukefirehawk/angel/tree/master/packages/test
+tridev_test,1.0.0,109,93,0,0,>=0.12.3 <0.14.0,regular,>=2.12.0 <3.0.0,https://github.com/tridev-dart/tridev
+flutter_lwa_platform_interface,2.1.0,46,71,41,0,^0.12.10,regular,>=2.12.0 <3.0.0,https://github.com/ayvazj/flutter_lwa
+endaft_core,0.0.3-dev.45,97,71,22,0,^0.12.12,regular,>=2.18.0 <3.0.0,https://github.com/endaft/endaft-core
diff --git a/pkgs/corpus/lib/pub.dart b/pkgs/corpus/lib/pub.dart
index 42a5b24..9c89f4c 100644
--- a/pkgs/corpus/lib/pub.dart
+++ b/pkgs/corpus/lib/pub.dart
@@ -43,17 +43,6 @@
);
}
- Future<List<String>> dependenciesOf(
- String packageName, {
- int? limit,
- }) async {
- return await _packageNamesForSearch(
- 'dependency:$packageName',
- limit: limit,
- sort: 'top',
- ).toList();
- }
-
Future<PackageInfo> getPackageInfo(String pkgName) async {
final json = await _getJson(Uri.https('pub.dev', 'api/packages/$pkgName'));
@@ -104,44 +93,6 @@
}
}
- Stream<String> _packageNamesForSearch(
- String query, {
- int page = 1,
- int? limit,
- String? sort,
- }) async* {
- final uri = Uri.parse('https://pub.dev/api/search');
-
- var count = 0;
-
- for (;;) {
- final targetUri = uri.replace(queryParameters: {
- if (query.isNotEmpty) 'q': query,
- 'page': page.toString(),
- if (sort != null) 'sort': sort,
- });
-
- final map = await _getJson(targetUri);
-
- for (var packageName in (map['packages'] as List)
- .cast<Map<String, dynamic>>()
- .map((e) => e['package'] as String?)) {
- count++;
- yield packageName!;
- }
-
- if (map.containsKey('next')) {
- page = page + 1;
- } else {
- break;
- }
-
- if (limit != null && count >= limit) {
- break;
- }
- }
- }
-
Future<Map<String, dynamic>> _getJson(Uri uri) async {
final result = await _client.get(uri);
if (result.statusCode == 200) {
@@ -285,8 +236,8 @@
PackageScore.from(this.json);
- int get grantedPoints => json['grantedPoints'] as int;
- int get maxPoints => json['maxPoints'] as int;
- int get likeCount => json['likeCount'] as int;
- double get popularityScore => json['popularityScore'] as double;
+ int get grantedPoints => json['grantedPoints'] as int? ?? 0;
+ int get maxPoints => json['maxPoints'] as int? ?? 140;
+ int get likeCount => json['likeCount'] as int? ?? 0;
+ double get popularityScore => json['popularityScore'] as double? ?? 0.0;
}
diff --git a/pkgs/corpus/lib/api_usage.dart b/pkgs/corpus/lib/usage.dart
similarity index 100%
rename from pkgs/corpus/lib/api_usage.dart
rename to pkgs/corpus/lib/usage.dart
