tools/testing/perf_testing/calc_and_send_mail.py

#!/usr/bin/python

# Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

import create_graph
from create_graph import BROWSER_CORRECTNESS
from create_graph import BROWSER_PERF
from create_graph import CL_PERF
from create_graph import COMMAND_LINE
from create_graph import CORRECTNESS
from create_graph import FROG
from create_graph import FROG_MEAN
from create_graph import TIME_SIZE
from create_graph import V8_AND_FROG

import get_current_stats
import os
from os.path import dirname, abspath
import pickle
import sys

"""Compare the current performance statistics with the statistics 24 hours ago,
and send out an email summarizing the differences."""

def calculate_stats():
  """Compare the numbers that were available at the start of the day to what the
  current numbers are.

  Returns:
    A string providing an update on the latest perfromance numbers."""
  test_runner_dict = pickle.load(open(get_current_stats.PICKLE_FILENAME, 'r'))
  test_runner_dict = get_current_stats.populate_stats_dict(test_runner_dict)

  browser_perf = test_runner_dict[BROWSER_PERF]
  time_size = test_runner_dict[TIME_SIZE]
  cl = test_runner_dict[CL_PERF]
  correctness = test_runner_dict[BROWSER_CORRECTNESS]
  output = summary_stats(browser_perf, time_size, cl, correctness)
  output += specific_stats(browser_perf, time_size, cl)
  return output

def summary_stats(browser_perf, time_size, cl, correctness):
  """Return the summarized stats report.

  Args:
    browser_perf: BrowserPerformanceTestRunner object. Holds browser perf stats.
    time_size: CompileTimeSizeTestRunner object. 
    cl: CommandLinePerformanceTestRunner object.
    correctness: BrowserCorrectnessTestRunner object.
  """
  output = "Summary of changes in the last 24 hours: \n\nBrowser " + \
      "performance: (revision %d)\n" % \
      browser_perf.revision_dict[create_graph.get_browsers()[0]][FROG]\
      [browser_perf.values_list[0]][1]
  for browser in create_graph.get_browsers():
    geo_mean_list = browser_perf.values_dict[browser][FROG][FROG_MEAN]
    # TODO(efortuna): deal with the fact that the latest of all browsers may not
    # be available.
    output += "  %s%s\n" % ((browser + ':').ljust(25), 
        str(geo_mean_list[1] - geo_mean_list[0]).rjust(10))

  output += "\nCompile Size and Time: (revision %d)\n" % \
      time_size.revision_dict[COMMAND_LINE][FROG][time_size.values_list[0]][1]
  for metric in time_size.values_list:
    metric_list = time_size.values_dict[COMMAND_LINE][FROG][metric]
    output += "  %s%s\n" % ((metric + ':').ljust(25), 
        str(metric_list[1] - metric_list[0]).rjust(10))

  output += "\nPercentage of language tests passing (revision %d)\n" % \
      correctness.revision_dict['chrome'][FROG][correctness.values_list[0]][1]
  for browser in create_graph.get_browsers():
    num_correct = correctness.values_dict[browser][FROG][CORRECTNESS]
    output += "  %s%s%%  more passing\n" % ((browser + ':').ljust(25), 
        str(num_correct[1] - num_correct[0]).rjust(10))

  output += "\nCommandline performance: (revision %d)\n" % \
      cl.revision_dict[COMMAND_LINE][FROG][cl.values_list[0]][1]
  for benchmark in cl.values_list:
    bench_list = cl.values_dict[COMMAND_LINE][FROG][benchmark]
    output += "  %s%s\n" % ((benchmark + ':').ljust(25), 
        str(bench_list[1] - bench_list[0]).rjust(10))
  return output

def specific_stats(browser_perf, time_size, cl):
  """Return a string detailing all of the gory details and specifics on 
  benchmark numbers and individual benchmark changes.

  Args:
    browser_perf: BrowserPerformanceTestRunner object. Holds browser perf stats.
    time_size: CompileTimeSizeTestRunner object. 
    cl: CommandLinePerformanceTestRunner object.
  """
  output = "\n\n---------------------------------------------\nThe latest " + \
      "current raw numbers (and changes) for those " + \
      "interested:\nBrowser performance:\n"
  for v8_or_frog in V8_AND_FROG:
    for browser in create_graph.get_browsers():
      output += "  %s %s:\n" % (browser, v8_or_frog)
      for benchmark in create_graph.get_benchmarks():
        bench_list = browser_perf.values_dict[browser][v8_or_frog][benchmark]
        output += "    %s %s%s\n" % ((benchmark + ':').ljust(25), 
            str(bench_list[1]).rjust(10), get_amount_changed(bench_list))

  output += "\nCompile Size and Time for frog:\n" 
  for metric in time_size.values_list:
    metric_list = time_size.values_dict[COMMAND_LINE][FROG][metric]
    output += "    %s %s%s\n" % ((metric + ':').ljust(25), 
        str(metric_list[1]).rjust(10), get_amount_changed(metric_list))

  output += "\nCommandline performance:\n"
  for v8_or_frog in V8_AND_FROG:
    output += '  %s:\n' % v8_or_frog
    for benchmark in cl.values_list:
      bench_list = cl.values_dict[COMMAND_LINE][v8_or_frog][benchmark]
      output += "    %s %s%s\n" % ((benchmark + ':').ljust(25), 
          str(bench_list[1]).rjust(10), get_amount_changed(bench_list))

  return output

def get_amount_changed(values_tuple):
  """Return a formatted string indicating the amount of change (positive or
  negative) in the benchmark since the last run.

  Args:
    values_tuple: the tuple of values we are comparing the difference 
    between."""
  difference = values_tuple[1] - values_tuple[0]
  prefix = '+' 
  if difference < 0:
    prefix = '-'
  return ("(%s%s)" % (prefix, str(difference))).rjust(10)


def main():
  stats = calculate_stats()
  print stats

if __name__ == '__main__':
  main()