Implement keyword subscriptions in the GitHub label notifier
We store per-repository list of keywords in the Firestore - if any of the
keywords are found in the body of the newly opened issue then this is
considered as if the issue was labeled with a specified label.
We plan to use this to get early notifications about dart related issues
filed in the flutter/flutter repository: having words like `gen_snapshot`,
SIGSEGV, etc to notify everybody subscribed to dependency: dart label.
We are not planning to have any UI for editing keyword subscriptions for now.
Change-Id: Ie22342f0a34a46c3165c21f65998439cc5f5d581
Reviewed-on: https://dart-review.googlesource.com/c/dart_ci/+/128861
Reviewed-by: Alexander Thomas <athom@google.com>
diff --git a/github-label-notifier/functions/README.md b/github-label-notifier/functions/README.md
index 0dc7e28..d31dc4e 100644
--- a/github-label-notifier/functions/README.md
+++ b/github-label-notifier/functions/README.md
@@ -62,4 +62,20 @@
Subscriptions are intended to be managed by users themselves so they are indexed
by Firebase UID issued by Firebase authentification.
+Additionally this collection contains configuration of keyword subscriptions
+for specific repositories at `github-label-subscriptions/{repositoryName}`,
+where `{repositoryName}` is full repository name with `/` replaced with `$`.
+
+```proto
+// Represents a subscription to keywords inside issue body. If the match is
+// found then this is treated as if the issue is labeled with the given label.
+message KeywordSubscription {
+ string label = 1;
+ repeated string keywords = 2;
+}
+```
+
+Note: security rules prevent editing subscriptions by anybody - so they
+can only be changed via Firebase Console UI.
+
Mails are sent via SendGrid.
diff --git a/github-label-notifier/functions/lib/subscriptions_db.dart b/github-label-notifier/functions/lib/subscriptions_db.dart
index 4b8bf26..1b22136 100644
--- a/github-label-notifier/functions/lib/subscriptions_db.dart
+++ b/github-label-notifier/functions/lib/subscriptions_db.dart
@@ -6,6 +6,7 @@
library github_label_notifier.subscriptions_db;
import 'package:firebase_admin_interop/firebase_admin_interop.dart';
+import 'package:meta/meta.dart';
final _app = FirebaseAdmin.instance.initializeApp();
final _firestore = _app.firestore();
@@ -13,12 +14,12 @@
void ensureInitialized() {
// For some reason the very first query takes very long time (>10s).
// We work around that by making a dummy query at the startup.
- lookupSubscriberEmails('', '');
+ lookupLabelSubscriberEmails('', '');
}
/// Return a list of emails subscribed to the given label in the given
/// repository.
-Future<List<String>> lookupSubscriberEmails(
+Future<List<String>> lookupLabelSubscriberEmails(
String repositoryName, String labelName) async {
final labelId = '${repositoryName}:${labelName}';
@@ -30,3 +31,42 @@
for (final doc in subscriptions.documents) doc.data.getString('email')
];
}
+
+/// Represents subscription to particular keywords in the issue body.
+///
+/// When a match is found we treat it as if the issue was labeled with the
+/// given label.
+class KeywordSubscription {
+ final String label;
+ final List<String> keywords;
+
+ KeywordSubscription({@required this.label, @required List<String> keywords})
+ : keywords = keywords.where(_isOk).toList();
+
+ /// Checks if the given string contains match to any of the keywords
+ /// and returns the matched one.
+ String match(String body) {
+ return _makeRegExp().firstMatch(body)?.group(0);
+ }
+
+ RegExp _makeRegExp() => RegExp(keywords.join('|'), caseSensitive: false);
+
+ static final _keywordRegExp = RegExp(r'^[\w_/]+$');
+ static bool _isOk(String keyword) => _keywordRegExp.hasMatch(keyword);
+}
+
+/// Get keyword subscriptions for the given repository.
+Future<KeywordSubscription> lookupKeywordSubscription(
+ String repositoryName) async {
+ final sanitizedRepositoryName = repositoryName.replaceAll('/', r'$');
+ final subscriptions = await _firestore
+ .document('github-label-subscriptions/$sanitizedRepositoryName')
+ .get();
+
+ if (!subscriptions.exists) {
+ return null;
+ }
+ return KeywordSubscription(
+ label: subscriptions.data.getString('label'),
+ keywords: subscriptions.data.getList('keywords').cast<String>());
+}
diff --git a/github-label-notifier/functions/node/index.dart b/github-label-notifier/functions/node/index.dart
index c7232d0..77f6481 100644
--- a/github-label-notifier/functions/node/index.dart
+++ b/github-label-notifier/functions/node/index.dart
@@ -71,7 +71,8 @@
};
final issueActionHandlers = <String, GitHubEventHandler>{
- 'labeled': onLabeled,
+ 'labeled': onIssueLabeled,
+ 'opened': onIssueOpened,
};
/// Handler for the 'labeled' issue event which triggers whenever an
@@ -79,11 +80,12 @@
///
/// The handler will send mails to all users that subscribed to a
/// particular label.
-Future<void> onLabeled(Map<String, dynamic> event) async {
+Future<void> onIssueLabeled(Map<String, dynamic> event) async {
final labelName = event['label']['name'];
final repositoryName = event['repository']['full_name'];
- final emails = await db.lookupSubscriberEmails(repositoryName, labelName);
+ final emails =
+ await db.lookupLabelSubscriberEmails(repositoryName, labelName);
if (emails.isEmpty) {
return;
}
@@ -110,6 +112,9 @@
Reported by ${issueReporterUsername}
Labeled ${labelName} by ${senderUser}
+
+--
+Sent by dart-github-label-notifier.web.app
''',
html: '''
<p><strong><a href="${issueUrl}">${escape(issueTitle)}</a> (${escape(repositoryName)}#${escape(issueNumber)})</strong></p>
@@ -120,6 +125,58 @@
''');
}
+/// Handler for the 'opened' issue event which triggers whenever a new issue
+/// is opened at the repository.
+///
+/// The handler will search the body of the open issue for specific keywords
+/// and send emails to all subscribers to a specific label.
+Future<void> onIssueOpened(Map<String, dynamic> event) async {
+ final repositoryName = event['repository']['full_name'];
+ final subscription = await db.lookupKeywordSubscription(repositoryName);
+
+ final match = subscription?.match(event['issue']['body']);
+ if (match == null) {
+ return;
+ }
+
+ final subscribers =
+ await db.lookupLabelSubscriberEmails(repositoryName, subscription.label);
+
+ final issueData = event['issue'];
+
+ final issueTitle = issueData['title'];
+ final issueNumber = issueData['number'];
+ final issueUrl = issueData['html_url'];
+ final issueReporterUsername = issueData['user']['login'];
+ final issueReporterUrl = issueData['user']['html_url'];
+
+ final escape = htmlEscape.convert;
+
+ await sendgrid.sendMultiple(
+ from: 'noreply@dart.dev',
+ to: subscribers,
+ subject: '[$repositoryName] ${issueTitle} (#${issueNumber})',
+ text: '''
+${issueUrl}
+
+Reported by ${issueReporterUsername}
+
+Matches keyword: ${match}
+
+You are getting this mail because you are subscribed to label ${subscription.label}.
+--
+Sent by dart-github-label-notifier.web.app
+''',
+ html: '''
+<p><strong><a href="${issueUrl}">${escape(issueTitle)}</a> (${escape(repositoryName)}#${escape(issueNumber)})</strong></p>
+<p>Reported by <a href="${issueReporterUrl}">${escape(issueReporterUsername)}</a></p>
+<p>Matches keyword: <b>${match}</b></p>
+<p>You are getting this mail because you are subscribed to label ${subscription.label}</p>
+<hr>
+<p>Sent by <a href="https://dart-github-label-notifier.web.app/">GitHub Label Notifier</a></p>
+''');
+}
+
class WebHookError {
final int statusCode;
final String message;
diff --git a/github-label-notifier/functions/node/index.test.dart b/github-label-notifier/functions/node/index.test.dart
index 5fd84d4..c9e4337 100644
--- a/github-label-notifier/functions/node/index.test.dart
+++ b/github-label-notifier/functions/node/index.test.dart
@@ -43,17 +43,29 @@
'email': 'first@example.com',
'subscriptions': [
'dart-lang/webhook-test:some-label',
- 'dart-lang/webhook-test:feature'
- ]
+ 'dart-lang/webhook-test:feature',
+ ],
}));
await subscriptions.add(DocumentData.fromMap({
'email': 'second@example.com',
'subscriptions': [
'dart-lang/webhook-test:bug',
- 'dart-lang/webhook-test:feature'
- ]
+ 'dart-lang/webhook-test:feature',
+ 'dart-lang/webhook-test:special-label',
+ ],
}));
+ await subscriptions
+ .document('dart-lang\$webhook-test')
+ .setData(DocumentData.fromMap({
+ 'keywords': [
+ 'jit',
+ 'aot',
+ 'third_party/dart',
+ ],
+ 'label': 'special-label',
+ }));
+
// Start mock SendGrid server at the address/port specified in
// SENDGRID_MOCK_SERVER environment variable.
// This server will simply record headers and bodies of all requests
@@ -151,6 +163,26 @@
}
};
+ Map<String, dynamic> makeIssueOpenedEvent(
+ {int number = 1, String repositoryName = 'dart-lang/webhook-test'}) =>
+ {
+ 'action': 'opened',
+ 'issue': {
+ 'html_url':
+ 'https://github.com/dart-lang/webhook-test/issues/${number}',
+ 'number': number,
+ 'title': 'TEST ISSUE TITLE',
+ 'body': 'This is an amazing aot solution',
+ 'user': {
+ 'login': 'hest',
+ 'html_url': 'https://github.com/hest',
+ },
+ },
+ 'repository': {
+ 'full_name': 'dart-lang/webhook-test',
+ },
+ };
+
test('signing', () {
expect(signEvent(makeLabeledEvent(labelName: 'bug')),
equals('sha1=76af51cdb9c7a43b146d4df721ac8f83e53182e5'));
@@ -252,6 +284,31 @@
}
]));
});
+
+ test('ok - single keyword subscriber', () async {
+ final rs = await sendEvent(body: makeIssueOpenedEvent());
+ expect(rs.statusCode, equals(HttpStatus.ok));
+ expect(sendgridRequests.length, equals(1));
+
+ final rq = sendgridRequests.first;
+ expect(rq.headers['authorization'],
+ equals('Bearer ' + Platform.environment['SENDGRID_SECRET']));
+ expect(rq.body['subject'], contains('#1'));
+ expect(rq.body['subject'], contains('TEST ISSUE TITLE'));
+ expect(
+ rq.body['personalizations'],
+ unorderedEquals([
+ {
+ 'to': [
+ {'email': 'second@example.com'}
+ ]
+ }
+ ]));
+ expect(
+ rq.body['content']
+ .firstWhere((c) => c['type'] == 'text/plain')['value'],
+ contains('Matches keyword: aot'));
+ });
}
/// Helper method to convert a [Buffer] to a [String].
diff --git a/github-label-notifier/ui/lib/app_component.dart b/github-label-notifier/ui/lib/app_component.dart
index e896c0a..5b2f7f9 100644
--- a/github-label-notifier/ui/lib/app_component.dart
+++ b/github-label-notifier/ui/lib/app_component.dart
@@ -13,7 +13,7 @@
import 'src/services/github_service.dart';
import 'src/services/subscription_service.dart';
-const repositoriesWithInstalledHook = ['dart-lang/sdk', 'dart-lang/language'];
+const repositoriesWithInstalledHook = ['dart-lang/sdk', 'flutter/flutter'];
@Component(
selector: 'my-app',
