tools/html_json_doc/lib/html_to_json.dart

// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/**
 * Library for extracting the documentation comments from files generated by
 * the HTML library. The comments are stored in a JSON file.
 *
 * Comments must be in either the block style with leading *s:
 *
 *     /**
 *      * Comment here.
 *      */
 *
 * Or the triple-slash style:
 *
 *     /// Docs go here.
 *     /// And here.
 *
 * Each member that is to be documented should be preceeded by a meta-comment
 * containing the string `@docsEditable` such as:
 *
 *     /// @docsEditable
 */
library html_to_json;

import 'dart:json';
import 'dart:io';
import 'dart:async';


/// True if any errors were triggered through the conversion.
bool _anyErrors = false;


/**
 * Convert files on [htmlPath] and write JSON to [jsonPath].
 */
Future<bool> convert(Path htmlPath, Path jsonPath) {
  var completer = new Completer();

  // TODO(amouravski): make this transform once I know what I want this file to
  // return.
  _convertFiles(htmlPath).then((convertedJson) {
    final jsonFile = new File.fromPath(jsonPath);
    var writeJson = convertedJson;

    if (jsonFile.existsSync()) {
      writeJson = _mergeJsonAndFile(convertedJson, jsonFile);
    }

    var outputStream = jsonFile.openOutputStream();
    outputStream.writeString(prettyPrintJson(writeJson));

    outputStream.onNoPendingWrites = () {
      completer.complete(_anyErrors);
    };

    outputStream.onClosed = () {
      completer.complete(_anyErrors);
    };

    outputStream.onError = completer.completeError;
  });

  return completer.future;
}


/**
 * Convert all files on [htmlPath].
 *
 * Returns a future that completes to the converted JSON object.
 */
Future<Object> _convertFiles(Path htmlPath) {
  var completer = new Completer();

  List<Future> fileFutures = [];

  // Get a list of all HTML dart files.
  // TODO(amouravski): discriminate .dart files.
  final htmlDir = new Directory.fromPath(htmlPath);
  final lister = htmlDir.list(recursive: false);

  lister.onFile = (String path) {
    final name = new Path(path).filename;

    // Ignore private classes.
    if (name.startsWith('_')) return;

    // Ignore non-dart files.
    if (!name.endsWith('.dart')) return;

    File file = new File(path);

    // TODO(amouravski): Handle missing file.
    if (!file.existsSync()) {
      print('ERROR: cannot find file $path');
      _anyErrors = true;
      return;
    }

    fileFutures.add(_convertFile(file));
  };


  // Combine all JSON objects
  lister.onDone = (_) {
    Futures.wait(fileFutures).then((jsonList) {
      var convertedJson = {};
      jsonList.forEach((json) {
        final k = json.keys[0];
        convertedJson.putIfAbsent(k, () => json[k]);
      });
      completer.complete(convertedJson);
    });
  };

  // TODO(amouravski): add more error handling.

  return completer.future;
}


/**
 * Convert a single file to JSON docs.
 *
 * Returns a map with one entry whose key is the file name and whose value is
 * the list of comment lines.
 */
Future<Map> _convertFile(File file) {
  var completer = new Completer();

  var comments = {};

  // Find all /// @docsEditable annotations.
  InputStream file_stream = file.openInputStream();
  StringInputStream inputLines = new StringInputStream(file_stream);

  // TODO(amouravski): Re-write as file.readAsLine().thin((lines) {...}
  inputLines.onLine = () {
    var comment = <String>[];

    var docCommentFound = false;
    String line;
    while ((line = inputLines.readLine()) != null) {
      var trimmedLine = line.trim();

      // Sentinel found. Process the comment block.
      if (trimmedLine.startsWith('///') &&
          trimmedLine.contains('@docsEditable')) {
        if (docCommentFound == true) {
          var nextLine = inputLines.readLine();

          if (nextLine == null) return false;

          var lineObject = {};

          if (comments[nextLine] != null) {
            print('WARNING: duplicate line ${nextLine} found in'
                '${new Path(file.fullPathSync()).filename}');
          }
          comments.putIfAbsent(nextLine, () => comment);
        }

        // Reset.
        docCommentFound = false;
        comment = <String>[];
      } else if ( // Start a comment block.
          trimmedLine.startsWith('/**') ||
          trimmedLine.startsWith('///')) {
        docCommentFound = true;
        comment.add(line);
      } else if (docCommentFound &&
          // TODO(amouravski): This will barf on:
          // /// blah
          // *
          (trimmedLine.startsWith('*') || trimmedLine.startsWith('///'))) {
        comment.add(line);
      } else {
        // Reset if we're not in a comment.
        docCommentFound = false;
        comment = <String>[];
      }
    }
  };

  inputLines.onClosed = () {
    var jsonObject = {};
    jsonObject[new Path(file.fullPathSync()).filename] = comments;
    completer.complete(jsonObject);
  };

  // TODO(amouravski): better error handling.

  return completer.future;
}


/**
 * Merge the new JSON object and the existing file.
 */
Object _mergeJsonAndFile(Object json, File file) {
  var completer = new Completer();

  var fileJson = {};
  var jsonRead = file.readAsStringSync();

  if (jsonRead == '') {
    print('WARNING: no data read from '
        '${new Path(file.fullPathSync()).filename}');
    _anyErrors = true;
  } else {
    fileJson = JSON.parse(jsonRead);
  }
  return _mergeJson(json, fileJson);
}


/**
 * Merge two JSON objects, such that the returned JSON object is the
 * union of both.
 *
 * Each JSON must be a map, with each value being a map.
 */
Object _mergeJson(Object json1, Object json2) {
  if (json1 is Map && json2 is Map) {
    // Then check if [json2] contains any key form [json1], in which case
    // add all of the values from [json2] to the values of [json1].
    json2.forEach((k, v) {
      if (json1.containsKey(k)) {
        v.forEach((vk, vv) {
          if (json1[k].containsKey(vk) &&
              !_listsEqual(json1[k][vk],vv)) {
            // Assume that json1 is more current and take its data as opposed
            // to json2's.
            // TODO(amouravski): add better warning message and only if there's
            // a conflict.
            print('INFO: duplicate keys.');
            _anyErrors = false;
          } else {
            json1[k].putIfAbsent(vk, () => vv);
          }
        });
      } else {
        json1.putIfAbsent(k, () => v);
      }
    });
  } else {
    throw new ArgumentError('JSON objects must both be Maps');
  }

  // TODO(amouravski): more error handling.

  return json1;
}


/**
 * Tests for equality between two lists.
 *
 * This checks the first level of depth, so does not work for nested lists.
 */
bool _listsEqual(List list1, List list2) {
  return list1.every((e) => list2.contains(e)) &&
      list2.every((e) => list1.contains(e));
}


/**
 * Print JSON in a much nicer format.
 *
 * For example:
 *
 *     {"foo":["bar","baz"],"boo":{"far:"faz"}}
 *
 * becomes:
 *
 *     {
 *       "foo":
 *         [
 *           "bar",
 *           "baz"
 *         ],
 *       "boo":
 *         {
 *           "far":
 *             "faz"
 *         }
 *     }
 */
String prettyPrintJson(Object json, [String indentation = '']) {
  var output;

  if (json is List) {
    var recursiveOutput =
        Strings.join(json.map((e) =>
            prettyPrintJson(e, '$indentation  ')), ',\n');
    output = '$indentation[\n'
      '$recursiveOutput'
      '\n$indentation]';
  } else if (json is Map) {
    var keys = json.keys
      ..sort();

    // TODO(amouravski): No newline after :
    var mapList = keys.map((key) =>
        '$indentation${JSON.stringify(key)}:\n'
        '${prettyPrintJson(json[key], '$indentation  ')}');
    var recursiveOutput = Strings.join(mapList, ',\n');
    output = '$indentation{\n'
      '$recursiveOutput'
      '\n$indentation}';
  } else {
    output = '$indentation${JSON.stringify(json)}';
  }
  return output;
}