#!/usr/bin/env bash
# Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

# Script that collects progress metrics of the analyzer/FE integration.

# Metric 1: parser tests via fasta

# Run the suite to extract the total number of tests from the runner output.
# TODO(sigmund): don't require `dart` to be on the path.
total=$(dart pkg/analyzer/test/generated/parser_fasta_test.dart | \
  tail -1 | \
  sed -e "s/.*+\([0-9]*\)[^0-9].*All tests passed.*$/\1/")

# Count tests marked with the @failingTest annotation.
fail=$(cat pkg/analyzer/test/generated/parser_fasta_test.dart | \
  grep failingTest | wc -l)

pass_rate=$(bc <<< "scale=1; 100*($total-$fail)/$total")
echo "Parser-fasta tests:         $(($total - $fail))/$total ($pass_rate%)"

# Metric 2: analyzer tests with fasta enabled.

# Run analyzer tests forcing the fasta parser, then process the logged output to
# count the number of individual tests (a single test case in a test file) that
# are passing or failing.

echo "Analyzer tests files:"
logfile=$1
delete=0

# If a log file is provided on the command line, reuse it and don't run the
# suite again.
if [[ $logfile == '' ]]; then
  logfile=$(mktemp log-XXXXXX.txt)
  echo "  Log file: $logfile"
  # TODO: delete by default and stop logging the location of the file.
  # delete=1
  python tools/test.py -m release --checked --use-sdk \
     --vm-options="-DuseFastaParser=true" \
     pkg/analy > $logfile
fi;

pass=$(tail -1 $logfile | sed -e "s/.*+\s*\([0-9]*\) |.*$/\1/")
fail=$(tail -1 $logfile | sed -e "s/.* -\s*\([0-9]*\)\].*$/\1/")
pass_rate=$(bc <<< "scale=1; 100*$pass/($pass + $fail)")

echo "  Test files passing:       $pass/$(($pass + $fail)) ($pass_rate%)"

# Tests use package:test, which contains a summary line saying how many tests
# passed and failed. The line has this form:
#
#    MM:SS  +pp -ff: Some tests failed
#
# but also contains some escape sequences for color highlighting. The code below
# extracts the passing (pp) and failing (ff) numbers and tallies them up:
cat $logfile | \
  grep "Some tests failed" | \
  sed -e "s/.*+\([0-9]*\).* -\([0-9]*\).*/\1 \2/" | \
   awk '{
    pass += $1
    total += $1 + $2
  } END {
    printf ("  Individual tests passing: %d/%d (%.1f%)\n", \
      pass/2, total/2,(100 * pass / total))
  }'

if [[ $delete == 1 ]]; then
  echo "rm $logfile"
fi

# TODO: Add metric 3 - coverage of error codes