utils/apidoc/mdn/crawl.js

// TODO(jacobr): convert this file to Dart once Dart supports all of the
// nodejs functionality used here.  For example, search for all occurences of
// "http." and "fs."
var http = require('http');
var fs = require('fs');

try {
  fs.mkdirSync('output/crawl');
} catch (e) {
  // It doesn't matter if the directories already exist.
}

var domTypes = JSON.parse(fs.readFileSync('data/domTypes.json', 'utf8'));

var cacheData = {};

function scrape(filename, link) {
  console.log(link);
  var httpsPrefix = "https://";
  var prefix = 'https://developer.mozilla.org/';
  var notFoundPrefix = 'https://developer.mozilla.org/Article_not_found?uri=';
  if (link.indexOf(prefix) != 0 ) {
    throw "Unexpected url: " + link;
  }
  var scrapePath = "/search?q=cache:" + link;
  // We crawl content from googleusercontent.com so we don't have to worry about
  // crawler politeness like we would have to if scraping developer.mozilla.org
  // directly.
  var options = {
    host: 'webcache.googleusercontent.com',
    path: scrapePath,
    port: 80,
    method: 'GET'
  };

  var req = http.request(options, function(res) {
    res.setEncoding('utf8');
    var data='';

    res.on('data', function(d) {
      data += d;
    });
    var onClose = function(e) {
      console.log("Writing crawl result for " + link);
      fs.writeFileSync("output/crawl/" + filename + ".html", data, 'utf8');
    }
    res.on('close', onClose);
    res.on('end', onClose);
  });
  req.end();

  req.on('error', function(e) {
    throw "Error " + e + " scraping " + link;
  });
}

for (var i = 0; i < domTypes.length; i++) {
  var type = domTypes[i];

  // Json containing the search results for the current type.
  var data = fs.readFileSync("output/search/" + type + ".json");
  json = JSON.parse(data);
  if (!('items' in json)) {
    console.warn("No search results for " + type);
    continue;
  }
  var items = json['items'];

  var entry = [];
  cacheData[type] = entry;

  // Hardcode the correct matching url for a few types where the search engine
  // gets the wrong answer.
  var link = null;
  if (type == 'Screen') {
    link = 'https://developer.mozilla.org/en/DOM/window.screen';
  } else if (type == 'Text') {
    link = 'https://developer.mozilla.org/en/DOM/Text';
  } else if (type == 'Touch') {
    link = 'https://developer.mozilla.org/en/DOM/Touch';
  } else if (type == 'TouchEvent' || type == 'webkitTouchEvent' || type == 'WebkitTouchEvent' || type == 'WebKitTouchEvent') {
    link = 'https://developer.mozilla.org/en/DOM/TouchEvent';
  } else if (type == 'HTMLSpanElement') {
    link = 'https://developer.mozilla.org/en/HTML/Element/span';
  } else if (type == 'HTMLPreElement') {
    link = 'https://developer.mozilla.org/en/HTML/Element/pre';
  } else if (type == 'HTMLFrameElement') {
    link = 'https://developer.mozilla.org/en/HTML/Element/frame';
  } else if (type == 'HTMLFrameSetElement') {
    link = 'https://developer.mozilla.org/en/HTML/Element/frameset';
  } else if (type == 'Geolocation') {
    link = 'https://developer.mozilla.org/en/nsIDOMGeolocation;'
  } else if (type == 'Notification') {
    link = 'https://developer.mozilla.org/en/DOM/notification';
  } else if (type == 'IDBDatabase') {
    link = 'https://developer.mozilla.org/en/IndexedDB/IDBDatabase'
  }
  if (link != null) {
    entry.push({index: 0, link: link, title: type});
    scrape(type + 0, link);
    continue;
  }

  for (j = 0; j < items.length; j++) {
    var item = items[j];
    var prefix = 'https://developer.mozilla.org/';
    var notFoundPrefix = 'https://developer.mozilla.org/Article_not_found?uri=';
    // Be optimistic and replace article not found links with links to where the
    // article should be.
    link = item['link'];
    if (link.indexOf(notFoundPrefix) == 0) {
      link = prefix + link.substr(notFoundPrefix.length);
    }

    entry.push({index: j, link: link, title: item['title']});
    scrape(type + j, link);
  }
}

fs.writeFileSync('output/crawl/cache.json', JSON.stringify(cacheData, null, ' '), 'utf8');