testing/benchmark/displaylist_benchmark_parser.py

#!/usr/bin/env python3
#
# Copyright 2013 The Flutter Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse
import csv
import json
import sys
import matplotlib.pyplot as plt  # pylint: disable=import-error
from matplotlib.backends.backend_pdf import PdfPages as pdfp  # pylint: disable=import-error


class BenchmarkResult:  # pylint: disable=too-many-instance-attributes

  def __init__(self, name, backend, time_unit, draw_call_count):
    self.name = name
    self.series = {}
    self.series_labels = {}
    self.backend = backend
    self.large_y_values = False
    self.y_limit = 200
    self.time_unit = time_unit
    self.draw_call_count = draw_call_count
    self.optional_values = {}

  def __repr__(self):
    return 'Name: % s\nBackend: % s\nSeries: % s\nSeriesLabels: % s\n' % (
        self.name, self.backend, self.series, self.series_labels
    )

  def add_data_point(self, family, xval, yval):
    if family not in self.series:
      self.series[family] = {'x': [], 'y': []}

    self.series[family]['x'].append(xval)
    self.series[family]['y'].append(yval)

    if yval > self.y_limit:
      self.large_y_values = True

  def add_optional_value(self, name, xval, yval):
    if name not in self.optional_values:
      self.optional_values[name] = {}

    self.optional_values[name][xval] = yval

  def set_family_label(self, family, label):
    # I'm not keying the main series dict off the family label
    # just in case we get data where the two aren't a 1:1 mapping
    if family in self.series_labels:
      assert self.series_labels[family] == label
      return

    self.series_labels[family] = label

  def plot(self):
    figures = []
    figures.append(plt.figure(dpi=1200, frameon=False, figsize=(11, 8.5)))

    for family in self.series:
      plt.plot(self.series[family]['x'], self.series[family]['y'], label=self.series_labels[family])

    plt.xlabel('Benchmark Seed')
    plt.ylabel('Time (' + self.time_unit + ')')

    title = ''
    # Crop the Y axis so that we can see what's going on at the lower end
    if self.large_y_values:
      plt.ylim((0, self.y_limit))
      title = self.name + ' ' + self.backend + ' (Cropped)'
    else:
      title = self.name + ' ' + self.backend

    if self.draw_call_count != -1:
      title += '\nDraw Call Count: ' + str(int(self.draw_call_count))

    plt.title(title)

    plt.grid(which='both', axis='both')

    plt.legend(fontsize='xx-small')
    plt.plot()

    if self.large_y_values:
      # Plot again but with the full Y axis visible
      figures.append(plt.figure(dpi=1200, frameon=False, figsize=(11, 8.5)))
      for family in self.series:
        plt.plot(
            self.series[family]['x'], self.series[family]['y'], label=self.series_labels[family]
        )

      plt.xlabel('Benchmark Seed')
      plt.ylabel('Time (' + self.time_unit + ')')
      title = self.name + ' ' + self.backend + ' (Complete)'

      if self.draw_call_count != -1:
        title += '\nDraw Call Count: ' + str(int(self.draw_call_count))

      plt.title(title)

      plt.grid(which='both', axis='both')

      plt.legend(fontsize='xx-small')
      plt.plot()

    return figures

  def write_csv(self, writer):
    # For now assume that all our series have the same x values
    # this is true for now, but may differ in the future with benchmark changes
    x_values = []
    y_values = []
    for family in self.series:
      x_values = ['x'] + self.series[family]['x']
      y_values.append([self.series_labels[family]] + self.series[family]['y'])

    for name in self.optional_values:
      column = [name]
      for key in self.optional_values[name]:
        column.append(self.optional_values[name][key])
      y_values.append(column)

    writer.writerow([self.name, self.draw_call_count])
    for line, _ in enumerate(x_values):
      row = [x_values[line]]
      for series, _ in enumerate(y_values):
        row.append(y_values[series][line])
      writer.writerow(row)


def main():
  parser = argparse.ArgumentParser()

  parser.add_argument(
      'filename', action='store', help='Path to the JSON output from Google Benchmark'
  )
  parser.add_argument(
      '-o',
      '--output-pdf',
      dest='output_pdf',
      action='store',
      default='output.pdf',
      help='Filename to output the PDF of graphs to.'
  )
  parser.add_argument(
      '-c',
      '--output-csv',
      dest='output_csv',
      action='store',
      default='output.csv',
      help='Filename to output the CSV data to.'
  )

  args = parser.parse_args()
  json_data = parse_json(args.filename)
  return process_benchmark_data(json_data, args.output_pdf, args.output_csv)


def error(message):
  print(message)
  sys.exit(1)


def extrac_attributes_label(benchmark_result):
  # Possible attribute keys are:
  #  AntiAliasing
  #  HairlineStroke
  #  StrokedStyle
  #  FilledStyle
  attributes = ['AntiAliasing', 'HairlineStroke', 'StrokedStyle', 'FilledStyle']
  label = ''

  for attr in attributes:
    try:
      if benchmark_result[attr] != 0:
        label += attr + ', '
    except KeyError:
      pass

  return label[:-2]


def process_benchmark_data(benchmark_json, output_pdf, output_csv):
  benchmark_results_data = {}

  for benchmark_result in benchmark_json:
    # Skip aggregate results
    if 'aggregate_name' in benchmark_result:
      continue

    benchmark_variant = benchmark_result['name'].split('/')
    # The final split is always `real_time` and can be discarded
    benchmark_variant.remove('real_time')

    splits = len(benchmark_variant)
    # First split is always the benchmark function name
    benchmark_name = benchmark_variant[0]
    # The last split is always the seeded value into the benchmark
    benchmark_seeded_value = benchmark_variant[splits - 1]
    # The second last split is always the backend
    benchmark_backend = benchmark_variant[splits - 2]
    # Time taken (wall clock time) for benchmark to run
    benchmark_real_time = benchmark_result['real_time']
    benchmark_unit = benchmark_result['time_unit']

    benchmark_family_index = benchmark_result['family_index']

    benchmark_family_label = ''
    if splits > 3:
      for i in range(1, splits - 2):
        benchmark_family_label += benchmark_variant[i] + ', '

    benchmark_family_attributes = extrac_attributes_label(benchmark_result)

    if benchmark_family_attributes == '':
      benchmark_family_label = benchmark_family_label[:-2]
    else:
      benchmark_family_label = benchmark_family_label + benchmark_family_attributes

    if 'DrawCallCount' in benchmark_result:
      benchmark_draw_call_count = benchmark_result['DrawCallCount']
    else:
      benchmark_draw_call_count = -1

    optional_keys = ['DrawCallCount_Varies', 'VerbCount', 'PointCount', 'VertexCount', 'GlyphCount']

    if benchmark_name not in benchmark_results_data:
      benchmark_results_data[benchmark_name] = BenchmarkResult(
          benchmark_name, benchmark_backend, benchmark_unit, benchmark_draw_call_count
      )

    for key in optional_keys:
      if key in benchmark_result:
        benchmark_results_data[benchmark_name].add_optional_value(
            key, benchmark_seeded_value, benchmark_result[key]
        )

    benchmark_results_data[benchmark_name].add_data_point(
        benchmark_family_index, benchmark_seeded_value, benchmark_real_time
    )
    benchmark_results_data[benchmark_name].set_family_label(
        benchmark_family_index, benchmark_family_label
    )

  pdf = pdfp(output_pdf)

  csv_file = open(output_csv, 'w')
  csv_writer = csv.writer(csv_file)

  for benchmark in benchmark_results_data:
    figures = benchmark_results_data[benchmark].plot()
    for fig in figures:
      pdf.savefig(fig)
    benchmark_results_data[benchmark].write_csv(csv_writer)
  pdf.close()


def parse_json(filename):
  try:
    json_file = open(filename, 'r')
  except:  # pylint: disable=bare-except
    error('Unable to load file.')

  try:
    json_data = json.load(json_file)
  except JSONDecodeError:  # pylint: disable=undefined-variable
    error('Invalid JSON. Unable to parse.')

  return json_data['benchmarks']


if __name__ == '__main__':
  sys.exit(main())