[jnigen] Summarizer fix class listing (https://github.com/dart-lang/jnigen/issues/272)

* Add tests for summarizer failure cases.
* Fix class listings
diff --git a/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/Main.java b/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/Main.java
index 07f4430..1eda460 100644
--- a/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/Main.java
+++ b/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/Main.java
@@ -7,16 +7,14 @@
 import com.github.dart_lang.jnigen.apisummarizer.disasm.AsmSummarizer;
 import com.github.dart_lang.jnigen.apisummarizer.doclet.SummarizerDoclet;
 import com.github.dart_lang.jnigen.apisummarizer.elements.ClassDecl;
-import com.github.dart_lang.jnigen.apisummarizer.util.InputStreamProvider;
-import com.github.dart_lang.jnigen.apisummarizer.util.JsonUtil;
-import com.github.dart_lang.jnigen.apisummarizer.util.Log;
-import com.github.dart_lang.jnigen.apisummarizer.util.SearchUtil;
+import com.github.dart_lang.jnigen.apisummarizer.util.*;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.LinkedHashMap;
 import java.util.List;
 import javax.tools.DocumentationTool;
 import javax.tools.JavaFileObject;
@@ -85,31 +83,42 @@
             ? Arrays.asList(options.classPath.split(File.pathSeparator))
             : List.of();
 
-    var classStreamProviders = new ArrayList<InputStreamProvider>();
-    var sourceFiles = new ArrayList<JavaFileObject>();
-    var notFound = new ArrayList<String>();
-
     var javaDoc = ToolProvider.getSystemDocumentationTool();
 
+    var sourceClasses = new LinkedHashMap<String, List<JavaFileObject>>();
+    var binaryClasses = new LinkedHashMap<String, List<InputStreamProvider>>();
+
     for (var qualifiedName : options.args) {
-      var found = false;
-      if (options.backend != Backend.ASM) {
-        var sources =
-            SearchUtil.findJavaSources(
-                qualifiedName, sourcePaths, javaDoc.getStandardFileManager(null, null, null));
-        if (sources.isPresent()) {
-          sourceFiles.addAll(sources.get());
-          found = true;
-        }
+      sourceClasses.put(qualifiedName, null);
+      binaryClasses.put(qualifiedName, null);
+    }
+
+    if (options.backend != Backend.ASM) {
+      ClassFinder.findJavaSources(
+          sourceClasses, sourcePaths, javaDoc.getStandardFileManager(null, null, null));
+    }
+
+    // remove found classes from binaryClasses, so that they don't need to be searched again.
+    // TODO: Tidy up this logic, move to ClassFinder class
+    for (var qualifiedName : options.args) {
+      if (sourceClasses.get(qualifiedName) != null) {
+        binaryClasses.remove(qualifiedName);
       }
-      if (options.backend != Backend.DOCLET && !found) {
-        var classes = SearchUtil.findJavaClasses(qualifiedName, classPaths);
-        if (classes.isPresent()) {
-          classStreamProviders.addAll(classes.get());
-          found = true;
-        }
+    }
+
+    if (options.backend != Backend.DOCLET) {
+      ClassFinder.findJavaClasses(binaryClasses, classPaths);
+    }
+
+    // remove duplicates (found as both source & binary), and determine if any class is not found.
+    var notFound = new ArrayList<String>();
+    for (var qualifiedName : options.args) {
+      var foundSource = sourceClasses.get(qualifiedName) != null;
+      var foundBinary = binaryClasses.get(qualifiedName) != null;
+      if (foundSource) {
+        binaryClasses.remove(qualifiedName);
       }
-      if (!found) {
+      if (!foundBinary && !foundSource) {
         notFound.add(qualifiedName);
       }
     }
@@ -119,12 +128,15 @@
       System.exit(1);
     }
 
+    var classStreamProviders = StreamUtil.flattenListValues(binaryClasses);
+    var sourceFiles = StreamUtil.flattenListValues(sourceClasses);
+
     switch (options.backend) {
       case DOCLET:
-        JsonUtil.writeJSON(runDoclet(javaDoc, sourceFiles, options), output);
+        JsonWriter.writeJSON(runDoclet(javaDoc, sourceFiles, options), output);
         break;
       case ASM:
-        JsonUtil.writeJSON(AsmSummarizer.run(classStreamProviders), output);
+        JsonWriter.writeJSON(AsmSummarizer.run(classStreamProviders), output);
         break;
       case AUTO:
         List<ClassDecl> decls = new ArrayList<>();
@@ -134,7 +146,7 @@
         if (!classStreamProviders.isEmpty()) {
           decls.addAll(AsmSummarizer.run(classStreamProviders));
         }
-        JsonUtil.writeJSON(decls, output);
+        JsonWriter.writeJSON(decls, output);
         break;
     }
   }
diff --git a/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/ClassFinder.java b/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/ClassFinder.java
new file mode 100644
index 0000000..2938022
--- /dev/null
+++ b/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/ClassFinder.java
@@ -0,0 +1,166 @@
+package com.github.dart_lang.jnigen.apisummarizer.util;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.util.*;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import javax.tools.JavaFileObject;
+import javax.tools.StandardJavaFileManager;
+
+public class ClassFinder {
+  public static <E> void findFilesInPath(
+      String searchPath,
+      String suffix,
+      Map<String, List<E>> classes,
+      Function<List<File>, List<E>> mapper) {
+
+    for (var binaryName : classes.keySet()) {
+      if (classes.get(binaryName) != null) {
+        continue;
+      }
+      var s = binaryName.replace(".", File.separator);
+      var f = new File(searchPath, s + suffix);
+      if (f.exists() && f.isFile()) {
+        classes.put(binaryName, mapper.apply(List.of(f)));
+      }
+
+      var d = new File(searchPath, s);
+      if (d.exists() && d.isDirectory()) {
+        var files = recursiveListFiles(d, file -> file.getName().endsWith(suffix));
+        classes.put(binaryName, mapper.apply(files));
+      }
+    }
+  }
+
+  public static <E> void findFilesInJar(
+      Map<String, List<E>> classes,
+      JarFile jar,
+      String suffix,
+      BiFunction<JarFile, List<ZipEntry>, List<E>> mapper) {
+    var entries =
+        jar.stream().map(JarEntry::getName).collect(Collectors.toCollection(TreeSet::new));
+    for (var binaryName : classes.keySet()) {
+      if (classes.get(binaryName) != null) {
+        continue;
+      }
+      var relativePath = binaryName.replace('.', '/');
+
+      var filePath = relativePath + suffix;
+      if (entries.contains(filePath)) {
+        var found = List.of(jar.getEntry(filePath));
+        classes.put(binaryName, mapper.apply(jar, found));
+      }
+
+      // Obtain set of all strings prefixed with relativePath + '/'
+      var dirPath = relativePath + '/';
+      var children =
+          entries.tailSet(dirPath).stream()
+              .takeWhile(e -> e.startsWith(dirPath))
+              .filter(e -> e.endsWith(suffix))
+              .map(jar::getEntry)
+              .collect(Collectors.toList());
+      if (!children.isEmpty()) {
+        var mapped = mapper.apply(jar, children);
+        classes.put(binaryName, mapped);
+      }
+    }
+  }
+
+  public static <T> void find(
+      Map<String, List<T>> classes,
+      List<String> searchPaths,
+      String suffix,
+      Function<List<File>, List<T>> fileMapper,
+      BiFunction<JarFile, List<ZipEntry>, List<T>> entryMapper) {
+    for (var searchPath : searchPaths) {
+      File searchFile = new File(searchPath);
+      if (searchFile.isDirectory()) {
+        findFilesInPath(searchPath, suffix, classes, fileMapper);
+      } else if (searchFile.isFile() && searchPath.endsWith(".jar")) {
+        var jarFile = ExceptionUtil.wrapCheckedException(JarFile::new, searchPath);
+        findFilesInJar(classes, jarFile, suffix, entryMapper);
+      }
+    }
+  }
+
+  private static List<JavaFileObject> getJavaFileObjectsFromFiles(
+      List<File> files, StandardJavaFileManager fm) {
+    var result = new ArrayList<JavaFileObject>();
+    fm.getJavaFileObjectsFromFiles(files).forEach(result::add);
+    return result;
+  }
+
+  private static List<JavaFileObject> getJavaFileObjectsFromJar(
+      JarFile jarFile, List<ZipEntry> entries) {
+    return StreamUtil.map(entries, (entry) -> new JarEntryFileObject(jarFile, entry));
+  }
+
+  private static List<InputStreamProvider> getInputStreamProvidersFromFiles(List<File> files) {
+    return StreamUtil.map(files, FileInputStreamProvider::new);
+  }
+
+  private static List<InputStreamProvider> getInputStreamProvidersFromJar(
+      JarFile jarFile, List<ZipEntry> entries) {
+    return StreamUtil.map(entries, entry -> new JarEntryInputStreamProvider(jarFile, entry));
+  }
+
+  public static void findJavaSources(
+      Map<String, List<JavaFileObject>> classes,
+      List<String> searchPaths,
+      StandardJavaFileManager fm) {
+    find(
+        classes,
+        searchPaths,
+        ".java",
+        files -> getJavaFileObjectsFromFiles(files, fm),
+        ClassFinder::getJavaFileObjectsFromJar);
+  }
+
+  public static void findJavaClasses(
+      Map<String, List<InputStreamProvider>> classes, List<String> searchPaths) {
+    find(
+        classes,
+        searchPaths,
+        ".class",
+        ClassFinder::getInputStreamProvidersFromFiles,
+        ClassFinder::getInputStreamProvidersFromJar);
+  }
+
+  /**
+   * Lists all files under given directory, which satisfy the condition of filter. <br>
+   * The order of listing will be deterministic.
+   */
+  public static List<File> recursiveListFiles(File file, FileFilter filter) {
+    if (!file.exists()) {
+      throw new RuntimeException("File not found: " + file.getPath());
+    }
+
+    if (!file.isDirectory()) {
+      return List.of(file);
+    }
+
+    // List files using a breadth-first traversal.
+    var files = new ArrayList<File>();
+    var queue = new ArrayDeque<File>();
+    queue.add(file);
+    while (!queue.isEmpty()) {
+      var dir = queue.poll();
+      var list = dir.listFiles(entry -> entry.isDirectory() || filter.accept(entry));
+      if (list == null) throw new IllegalArgumentException("File.listFiles returned null!");
+      Arrays.sort(list);
+      for (var path : list) {
+        if (path.isDirectory()) {
+          queue.add(path);
+        } else {
+          files.add(path);
+        }
+      }
+    }
+    return files;
+  }
+}
diff --git a/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/JsonUtil.java b/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/JsonWriter.java
similarity index 96%
rename from pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/JsonUtil.java
rename to pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/JsonWriter.java
index cf7e8a4..873a6be 100644
--- a/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/JsonUtil.java
+++ b/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/JsonWriter.java
@@ -8,7 +8,7 @@
 import java.io.OutputStream;
 import java.util.List;
 
-public class JsonUtil {
+public class JsonWriter {
   public static void writeJSON(List<ClassDecl> classes, OutputStream output) {
     var mapper = new ObjectMapper();
     Log.timed("Writing JSON");
diff --git a/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/SearchUtil.java b/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/SearchUtil.java
deleted file mode 100644
index 9f1611d..0000000
--- a/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/SearchUtil.java
+++ /dev/null
@@ -1,155 +0,0 @@
-package com.github.dart_lang.jnigen.apisummarizer.util;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.util.*;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.jar.JarFile;
-import java.util.stream.Collectors;
-import java.util.zip.ZipEntry;
-import javax.tools.JavaFileObject;
-import javax.tools.StandardJavaFileManager;
-
-public class SearchUtil {
-  public static Optional<List<File>> findFilesInPath(
-      String qualifiedName, String searchPath, String suffix) {
-    var s = qualifiedName.replace(".", File.separator);
-    var f = new File(searchPath, s + suffix);
-    if (f.exists() && f.isFile()) {
-      return Optional.of(List.of(f));
-    }
-
-    var d = new File(searchPath, s);
-    if (d.exists() && d.isDirectory()) {
-      return Optional.of(recursiveListFiles(d, file -> file.getName().endsWith(".java")));
-    }
-
-    return Optional.empty();
-  }
-
-  public static Optional<List<ZipEntry>> findFilesInJar(
-      String qualifiedName, JarFile jar, String suffix) {
-    String relativePath = qualifiedName.replace(".", "/");
-    var classEntry = jar.getEntry(relativePath + suffix);
-    if (classEntry != null) {
-      return Optional.of(List.of(classEntry));
-    }
-    var dirPath = relativePath.endsWith("/") ? relativePath : relativePath + "/";
-    var dirEntry = jar.getEntry(dirPath);
-    if (dirEntry != null && dirEntry.isDirectory()) {
-      var result =
-          jar.stream()
-              .map(je -> (ZipEntry) je)
-              .filter(
-                  entry -> {
-                    var name = entry.getName();
-                    return name.endsWith(suffix) && name.startsWith(dirPath);
-                  })
-              .collect(Collectors.toList());
-      return Optional.of(result);
-    }
-    return Optional.empty();
-  }
-
-  public static <T> Optional<List<T>> find(
-      String qualifiedName,
-      List<String> searchPaths,
-      String suffix,
-      Function<List<File>, List<T>> fileMapper,
-      BiFunction<JarFile, List<ZipEntry>, List<T>> entryMapper) {
-    for (var searchPath : searchPaths) {
-      File searchFile = new File(searchPath);
-      if (searchFile.isDirectory()) {
-        var result = findFilesInPath(qualifiedName, searchPath, suffix);
-        if (result.isPresent()) {
-          var mappedResult = fileMapper.apply(result.get());
-          return Optional.of(mappedResult);
-        }
-      }
-      if (searchFile.isFile() && searchPath.endsWith(".jar")) {
-        var jarFile = ExceptionUtil.wrapCheckedException(JarFile::new, searchPath);
-        var result = findFilesInJar(qualifiedName, jarFile, suffix);
-        if (result.isPresent()) {
-          var mappedResult = entryMapper.apply(jarFile, result.get());
-          return Optional.of(mappedResult);
-        }
-      }
-    }
-    return Optional.empty();
-  }
-
-  private static List<JavaFileObject> getJavaFileObjectsFromFiles(
-      List<File> files, StandardJavaFileManager fm) {
-    var result = new ArrayList<JavaFileObject>();
-    fm.getJavaFileObjectsFromFiles(files).forEach(result::add);
-    return result;
-  }
-
-  private static List<JavaFileObject> getJavaFileObjectsFromJar(
-      JarFile jarFile, List<ZipEntry> entries) {
-    return StreamUtil.map(entries, (entry) -> new JarEntryFileObject(jarFile, entry));
-  }
-
-  private static List<InputStreamProvider> getInputStreamProvidersFromFiles(List<File> files) {
-    return StreamUtil.map(files, FileInputStreamProvider::new);
-  }
-
-  private static List<InputStreamProvider> getInputStreamProvidersFromJar(
-      JarFile jarFile, List<ZipEntry> entries) {
-    return StreamUtil.map(entries, entry -> new JarEntryInputStreamProvider(jarFile, entry));
-  }
-
-  public static Optional<List<JavaFileObject>> findJavaSources(
-      String qualifiedName, List<String> searchPaths, StandardJavaFileManager fm) {
-    return find(
-        qualifiedName,
-        searchPaths,
-        ".java",
-        files -> getJavaFileObjectsFromFiles(files, fm),
-        SearchUtil::getJavaFileObjectsFromJar);
-  }
-
-  public static Optional<List<InputStreamProvider>> findJavaClasses(
-      String qualifiedName, List<String> searchPaths) {
-    return find(
-        qualifiedName,
-        searchPaths,
-        ".class",
-        SearchUtil::getInputStreamProvidersFromFiles,
-        SearchUtil::getInputStreamProvidersFromJar);
-  }
-
-  /**
-   * Lists all files under given directory, which satisfy the condition of filter. <br>
-   * The order of listing will be deterministic.
-   */
-  public static List<File> recursiveListFiles(File file, FileFilter filter) {
-    if (!file.exists()) {
-      throw new RuntimeException("File not found: " + file.getPath());
-    }
-
-    if (!file.isDirectory()) {
-      return List.of(file);
-    }
-
-    // List files using a breadth-first traversal.
-    var files = new ArrayList<File>();
-    var queue = new ArrayDeque<File>();
-    queue.add(file);
-    while (!queue.isEmpty()) {
-      var dir = queue.poll();
-      var list = dir.listFiles(entry -> entry.isDirectory() || filter.accept(entry));
-      if (list == null) throw new IllegalArgumentException("File.listFiles returned null!");
-      Arrays.sort(list);
-      for (var path : list) {
-        if (path.isDirectory()) {
-          queue.add(path);
-        } else {
-          files.add(path);
-        }
-      }
-    }
-    return files;
-  }
-}
diff --git a/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/StreamUtil.java b/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/StreamUtil.java
index 7dfa29b..30a8fa0 100644
--- a/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/StreamUtil.java
+++ b/pkgs/jnigen/java/src/main/java/com/github/dart_lang/jnigen/apisummarizer/util/StreamUtil.java
@@ -4,8 +4,7 @@
 
 package com.github.dart_lang.jnigen.apisummarizer.util;
 
-import java.util.Arrays;
-import java.util.List;
+import java.util.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -17,4 +16,11 @@
   public static <T, R> List<R> map(T[] array, Function<T, R> function) {
     return Arrays.stream(array).map(function).collect(Collectors.toList());
   }
+
+  public static <K, E> List<E> flattenListValues(Map<K, List<E>> map) {
+    return map.values().stream()
+        .filter(Objects::nonNull)
+        .flatMap(Collection::stream)
+        .collect(Collectors.toList());
+  }
 }
diff --git a/pkgs/jnigen/lib/src/generate_bindings.dart b/pkgs/jnigen/lib/src/generate_bindings.dart
index bc4deb0..d553657 100644
--- a/pkgs/jnigen/lib/src/generate_bindings.dart
+++ b/pkgs/jnigen/lib/src/generate_bindings.dart
@@ -9,6 +9,7 @@
 import 'bindings/excluder.dart';
 import 'bindings/linker.dart';
 import 'bindings/renamer.dart';
+import 'elements/elements.dart';
 import 'summary/summary.dart';
 import 'config/config.dart';
 import 'tools/tools.dart';
@@ -22,7 +23,16 @@
 
   await buildSummarizerIfNotExists();
 
-  final classes = await getSummary(config);
+  final Classes classes;
+
+  try {
+    classes = await getSummary(config);
+  } on SummaryParseException catch (e) {
+    if (e.stderr != null) {
+      printError(e.stderr);
+    }
+    log.fatal(e.message);
+  }
 
   final cBased = config.outputConfig.bindingsType == BindingsType.cBased;
   classes
diff --git a/pkgs/jnigen/lib/src/summary/summary.dart b/pkgs/jnigen/lib/src/summary/summary.dart
index a2f600e..31473a6 100644
--- a/pkgs/jnigen/lib/src/summary/summary.dart
+++ b/pkgs/jnigen/lib/src/summary/summary.dart
@@ -12,6 +12,16 @@
 import '../generate_bindings.dart';
 import '../logging/logging.dart';
 
+class SummaryParseException implements Exception {
+  final String? stderr;
+  final String message;
+  SummaryParseException(this.message) : stderr = null;
+  SummaryParseException.withStderr(this.stderr, this.message);
+
+  @override
+  String toString() => message;
+}
+
 /// A command based summary source which calls the ApiSummarizer command.
 /// [sourcePaths] and [classPaths] can be provided for the summarizer to find
 /// required dependencies. The [classes] argument specifies the fully qualified
@@ -93,6 +103,8 @@
 }
 
 Future<Classes> getSummary(Config config) async {
+  // This function is a potential entry point in tests, which set log level to
+  // warning.
   setLoggingLevel(config.logLevel);
   final summarizer = SummarizerCommand(
     sourcePath: config.sourcePath,
@@ -154,24 +166,30 @@
 
   Process process;
   Stream<List<int>> input;
+  final stopwatch = Stopwatch()..start();
   try {
     process = await summarizer.runProcess();
     input = process.stdout;
   } on Exception catch (e) {
-    log.fatal('Cannot obtain API summary: $e');
+    throw SummaryParseException('Cannot generate API summary: $e');
   }
-  final errorLog = StringBuffer();
-  collectOutputStream(process.stderr, errorLog);
+  final stderrBuffer = StringBuffer();
+  collectOutputStream(process.stderr, stderrBuffer);
   final stream = const JsonDecoder().bind(const Utf8Decoder().bind(input));
   dynamic json;
   try {
     json = await stream.single;
+    stopwatch.stop();
+    log.info('Parsing inputs took ${stopwatch.elapsedMilliseconds} ms');
   } on Exception catch (e) {
-    printError(errorLog);
-    log.fatal('Cannot parse summary: $e');
+    await process.exitCode;
+    throw SummaryParseException.withStderr(
+      stderrBuffer.toString(),
+      'Cannot generate summary: $e',
+    );
   }
   if (json == null) {
-    log.fatal('Expected JSON element from summarizer.');
+    throw SummaryParseException('Expected JSON element from summarizer.');
   }
   final list = json as List;
   final classes = Classes.fromJson(list);
diff --git a/pkgs/jnigen/test/summary_generation_test.dart b/pkgs/jnigen/test/summary_generation_test.dart
index 8a4eab1..0133bac 100644
--- a/pkgs/jnigen/test/summary_generation_test.dart
+++ b/pkgs/jnigen/test/summary_generation_test.dart
@@ -8,6 +8,7 @@
 @Tags(['summarizer_test'])
 
 import 'dart:io';
+import 'dart:math';
 
 import 'package:jnigen/src/config/config.dart';
 import 'package:jnigen/src/elements/elements.dart';
@@ -20,7 +21,7 @@
 
 import 'test_util/test_util.dart';
 
-void expectNonEmptySummary(Classes? classes) {
+void expectSummaryHasAllClasses(Classes? classes) {
   expect(classes, isNotNull);
   final decls = classes!.decls;
   expect(decls.entries.length, greaterThanOrEqualTo(javaFiles.length));
@@ -70,6 +71,13 @@
 final simplePackageDir = Directory(simplePackagePath);
 final javaFiles = findFilesWithSuffix(simplePackageDir, '.java');
 final javaClasses = javaFiles.map(getClassNameFromPath).toList();
+// remove individual class listings from one package,
+// and add the package name instead, for testing.
+const _removalPackage = 'com.github.dart_lang.jnigen.pkg2';
+final summarizerClassesSpec = [
+  ...javaClasses.where((e) => !e.startsWith('$_removalPackage.')),
+  _removalPackage,
+];
 
 Config getConfig({List<String>? sourcePath, List<String>? classPath}) {
   return Config(
@@ -80,86 +88,126 @@
         structure: OutputStructure.singleFile,
       ),
     ),
-    classes: javaClasses,
+    classes: summarizerClassesSpec,
     sourcePath: sourcePath?.map((e) => Uri.file(e)).toList(),
     classPath: classPath?.map((e) => Uri.file(e)).toList(),
     logLevel: Level.WARNING,
   );
 }
 
+final random = Random.secure();
+
+void testSuccessCase(String description, Config config) {
+  config.classes = summarizerClassesSpec;
+  test(description, () async {
+    final classes = await getSummary(config);
+    expectSummaryHasAllClasses(classes);
+  });
+}
+
+void testFailureCase(
+    String description, Config config, String nonExistingClass) {
+  test(description, () async {
+    final insertPosition = random.nextInt(config.classes.length + 1);
+    config.classes = summarizerClassesSpec.sublist(0, insertPosition) +
+        [nonExistingClass] +
+        summarizerClassesSpec.sublist(insertPosition);
+    try {
+      await getSummary(config);
+    } on SummaryParseException catch (e) {
+      expect(e.stderr, isNotNull);
+      expect(e.stderr!, stringContainsInOrder(["Not found", nonExistingClass]));
+      return;
+    }
+    throw AssertionError("No exception was caught");
+  });
+}
+
+void testAllCases({
+  List<String>? sourcePath,
+  List<String>? classPath,
+}) {
+  testSuccessCase(
+    '- valid config',
+    getConfig(sourcePath: sourcePath, classPath: classPath),
+  );
+  testFailureCase(
+    '- should fail with non-existing class',
+    getConfig(sourcePath: sourcePath, classPath: classPath),
+    'com.github.dart_lang.jnigen.DoesNotExist',
+  );
+  testFailureCase(
+    '- should fail with non-existing package',
+    getConfig(sourcePath: sourcePath, classPath: classPath),
+    'com.github.dart_lang.notexist',
+  );
+}
+
 void main() async {
   await checkLocallyBuiltDependencies();
-  late Directory tempDir;
-  setUpAll(() async {
-    tempDir = getTempDir("jnigen_summary_tests_");
-  });
+  final tempDir = getTempDir("jnigen_summary_tests_");
 
-  test('Test summary generation from compiled JAR', () async {
+  group('Test summary generation from compiled JAR', () {
     final targetDir = tempDir.createTempSync("compiled_jar_test_");
-    await compileJavaFiles(simplePackageDir, targetDir);
-    final classFiles = findFilesWithSuffix(targetDir, '.class');
     final jarPath = join(targetDir.absolute.path, 'classes.jar');
-    await createJar(
-        artifactDir: targetDir.path, artifacts: classFiles, jarPath: jarPath);
-    final config = getConfig(classPath: [jarPath]);
-    final summaryClasses = await getSummary(config);
-    expectNonEmptySummary(summaryClasses);
+    setUpAll(() async {
+      await compileJavaFiles(simplePackageDir, targetDir);
+      final classFiles = findFilesWithSuffix(targetDir, '.class');
+      await createJar(
+          artifactDir: targetDir.path, artifacts: classFiles, jarPath: jarPath);
+    });
+    testAllCases(classPath: [jarPath]);
   });
 
-  test('Test summary generation from source JAR', () async {
+  group('Test summary generation from source JAR', () {
     final targetDir = tempDir.createTempSync("source_jar_test_");
     final jarPath = join(targetDir.path, 'sources.jar');
-    await createJar(
-        artifactDir: simplePackageDir.path,
-        artifacts: javaFiles,
-        jarPath: jarPath);
-    final config = getConfig(sourcePath: [jarPath]);
-    final summaryClasses = await getSummary(config);
-    expectNonEmptySummary(summaryClasses);
+    setUpAll(() async {
+      await createJar(
+          artifactDir: simplePackageDir.path,
+          artifacts: javaFiles,
+          jarPath: jarPath);
+    });
+    testAllCases(sourcePath: [jarPath]);
   });
 
-  test('Test summary generation from source folder', () async {
-    final config = getConfig(sourcePath: [simplePackagePath]);
-    final summaryClasses = await getSummary(config);
-    expectNonEmptySummary(summaryClasses);
+  group('Test summary generation from source folder', () {
+    testAllCases(sourcePath: [simplePackagePath]);
   });
 
-  test('Test summary generation from compiled classes in directory', () async {
+  group('Test summary generation from compiled classes in directory', () {
     final targetDir = tempDir.createTempSync("compiled_classes_test_");
-    await compileJavaFiles(simplePackageDir, targetDir);
-    final config = getConfig(classPath: [targetDir.path]);
-    final summaryClasses = await getSummary(config);
-    expectNonEmptySummary(summaryClasses);
+    setUpAll(() => compileJavaFiles(simplePackageDir, targetDir));
+    testAllCases(classPath: [targetDir.path]);
   });
 
   // Test summary generation from combination of a source and class path
-  test('Test summary generation from combination', () async {
+  group('Test summary generation from combination', () {
     final targetDir = tempDir.createTempSync("combination_test_");
-
+    final classesJarPath = join(targetDir.path, 'classes.jar');
     // remove a class from source files and create a source JAR
     final sourceFiles = javaFiles.toList();
     sourceFiles.removeLast();
     final sourceJarPath = join(targetDir.path, 'sources.jar');
-    await createJar(
-      artifactDir: simplePackageDir.path,
-      artifacts: sourceFiles,
-      jarPath: sourceJarPath,
-    );
+    setUpAll(() async {
+      await createJar(
+        artifactDir: simplePackageDir.path,
+        artifacts: sourceFiles,
+        jarPath: sourceJarPath,
+      );
 
-    await compileJavaFiles(simplePackageDir, targetDir);
-    final classFiles = findFilesWithSuffix(targetDir, '.class');
-    final classesJarPath = join(targetDir.path, 'classes.jar');
-    await createJar(
-      artifactDir: targetDir.path,
-      artifacts: classFiles,
-      jarPath: classesJarPath,
-    );
-    final config = getConfig(
+      await compileJavaFiles(simplePackageDir, targetDir);
+      final classFiles = findFilesWithSuffix(targetDir, '.class');
+      await createJar(
+        artifactDir: targetDir.path,
+        artifacts: classFiles,
+        jarPath: classesJarPath,
+      );
+    });
+    testAllCases(
       classPath: [classesJarPath],
       sourcePath: [sourceJarPath],
     );
-    final summaryClasses = await getSummary(config);
-    expectNonEmptySummary(summaryClasses);
   });
 
   tearDownAll(() => deleteTempDir(tempDir));