commit 892d5b6bd2b0b65738983c9f3e398f067943daa7
author William Hesse <whesse@google.com> Wed May 17 13:45:32 2023 +0000
committer William Hesse <whesse@google.com> Wed May 17 13:45:32 2023 +0000
tree ea327a4d85d4de3cab01dcdbf55b229f938ab90a
parent af7d2e60170e5a511cfc258a79420e6c454de326

[builder] Add retries to Firestore document create calls

Also limit the number of retries on Firestore document write calls.

Bug: b/283056052
Change-Id: I74ad1e82725c6528740c52ade9d5b58a8ca81c97
Reviewed-on: https://dart-review.googlesource.com/c/dart_ci/+/304021
Reviewed-by: Jonas Termansen <sortie@google.com>

builder/lib/src/firestore.dart

diff --git a/builder/lib/src/firestore.dart b/builder/lib/src/firestore.dart
index b20dc9f..80f13f9 100644
--- a/builder/lib/src/firestore.dart
+++ b/builder/lib/src/firestore.dart
@@ -262,11 +262,24 @@
   }
 
   Future<Document> storeResult(Map<String, dynamic> result) async {
-    final document = Document()..fields = taggedMap(result);
-    final createdDocument = await firestore.projects.databases.documents
-        .createDocument(document, documents, 'results');
-    documentsWritten++;
-    return createdDocument;
+    for (var retries = 0; true; retries++) {
+      try {
+        final document = Document()..fields = taggedMap(result);
+        final createdDocument = await firestore.projects.databases.documents
+            .createDocument(document, documents, 'results');
+        documentsWritten++;
+        return createdDocument;
+      } catch (e) {
+        log('Failed creating document at path $documents/results.');
+        log('Retrying in 1000 ms.');
+        log('Document contents: ${jsonEncode(result)}\n');
+        log('$e');
+        if (retries > 2) {
+          rethrow;
+        }
+        await Future.delayed(Duration(milliseconds: 1000));
+      }
+    }
   }
 
   Future<bool> updateResult(
@@ -314,8 +327,8 @@
     return approved;
   }
 
-  Future retryCommit(Future<Write> Function() request) async {
-    while (true) {
+  Future<void> retryCommit(Future<Write> Function() request) async {
+    for (var retries = 0; true; retries++) {
       final write = await request();
       try {
         // Use commit instead of write to check for the precondition on the
@@ -325,7 +338,10 @@
         await firestore.projects.databases.documents.commit(request, database);
         return;
       } catch (e) {
-        log('Error while writing data: $e, retrying...');
+        if (retries > 10) {
+          rethrow;
+        }
+        log('Error while writing data: $e, retrying.');
         sleep(Duration(milliseconds: 100));
       }
     }