blob: 4bb2d12ed34d9835af35713c47553cb73ab39359 [file] [log] [blame]
# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import copy
import database
import idlparser
import logging
import multiprocessing
import os
import os.path
import re
from idlnode import *
_logger = logging.getLogger('databasebuilder')
# Used in source annotations to specify the parent interface declaring
# a displaced declaration. The 'via' attribute specifies the parent interface
# which implements a displaced declaration.
class DatabaseBuilderOptions(object):
"""Used in specifying options when importing new interfaces"""
def __init__(self,
source=None, source_attributes={},
idl_syntax -- the syntax of the IDL file that is imported.
idl_defines -- list of definitions for the idl gcc pre-processor
source -- the origin of the IDL file, used for annotating the
source_attributes -- this map of attributes is used as
annotation attributes.
rename_operation_arguments_on_merge -- if True, will rename
operation arguments when merging using the new name rather
than the old.
add_new_interfaces -- when False, if an interface is a new
addition, it will be ignored.
obsolete_old_declarations -- when True, if a declaration
from a certain source is not re-declared, it will be removed.
self.source = source
self.source_attributes = source_attributes
self.idl_syntax = idl_syntax
self.idl_defines = idl_defines
self.rename_operation_arguments_on_merge = \
self.add_new_interfaces = add_new_interfaces
self.obsolete_old_declarations = obsolete_old_declarations
def _load_idl_file(file_name, import_options):
"""Loads an IDL file into memory"""
idl_parser = idlparser.IDLParser(import_options.idl_syntax)
f = open(file_name, 'r')
content =
idl_ast = idl_parser.parse(
return IDLFile(idl_ast, file_name)
except SyntaxError, e:
raise RuntimeError('Failed to load file %s: %s'
% (file_name, e))
class DatabaseBuilder(object):
def __init__(self, database):
"""DatabaseBuilder is used for importing and merging interfaces into
the Database"""
self._database = database
self._imported_interfaces = []
self._impl_stmts = []
self.conditionals_met = set()
def _resolve_type_defs(self, idl_file):
type_def_map = {}
# build map
for type_def in idl_file.typeDefs:
if != # sanity check
type_def_map[] =
# use the map
for type_node in idl_file.all(IDLType):
while in type_def_map: = type_def_map[]
def _strip_ext_attributes(self, idl_file):
"""Strips unuseful extended attributes."""
for ext_attrs in idl_file.all(IDLExtAttrs):
# TODO: Decide which attributes are uninteresting.
def _rename_types(self, idl_file, import_options):
"""Rename interface and type names with names provided in the
options. Also clears scopes from scoped names"""
strip_modules = lambda name: name.split('::')[-1]
def rename_node(idl_node):
def rename_ext_attrs(ext_attrs_node):
for type_valued_attribute_name in ['Supplemental']:
if type_valued_attribute_name in ext_attrs_node:
value = ext_attrs_node[type_valued_attribute_name]
if isinstance(value, str):
ext_attrs_node[type_valued_attribute_name] = strip_modules(value)
map(rename_node, idl_file.all(IDLInterface))
map(rename_node, idl_file.all(IDLType))
map(rename_ext_attrs, idl_file.all(IDLExtAttrs))
def _annotate(self, interface, import_options):
"""Adds @ annotations based on the source and source_attributes
members of import_options."""
source = import_options.source
if not source:
def add_source_annotation(idl_node):
annotation = IDLAnnotation(
idl_node.annotations[source] = annotation
if ((isinstance(idl_node, IDLInterface) or
isinstance(idl_node, IDLMember)) and
annotation['suppressed'] = None
map(add_source_annotation, interface.parents)
map(add_source_annotation, interface.constants)
map(add_source_annotation, interface.attributes)
map(add_source_annotation, interface.operations)
def _sign(self, node):
"""Computes a unique signature for the node, for merging purposed, by
concatenating types and names in the declaration."""
if isinstance(node, IDLType):
res =
if res.startswith('unsigned '):
res = res[len('unsigned '):]
return res
res = []
if isinstance(node, IDLInterface):
res = ['interface',]
elif isinstance(node, IDLParentInterface):
res = ['parent', self._sign(node.type)]
elif isinstance(node, IDLOperation):
res = ['op']
for special in node.specials:
if is not None:
for arg in node.arguments:
elif isinstance(node, IDLAttribute):
res = []
if node.is_read_only:
elif isinstance(node, IDLConstant):
res = []
raise TypeError("Can't sign input of type %s" % type(node))
return ':'.join(res)
def _build_signatures_map(self, idl_node_list):
"""Creates a hash table mapping signatures to idl_nodes for the
given list of nodes"""
res = {}
for idl_node in idl_node_list:
sig = self._sign(idl_node)
if sig is None:
if sig in res:
raise RuntimeError('Warning: Multiple members have the same '
'signature: "%s"' % sig)
res[sig] = idl_node
return res
def _get_parent_interfaces(self, interface):
"""Return a list of all the parent interfaces of a given interface"""
res = []
def recurse(current_interface):
if current_interface in res:
for parent in current_interface.parents:
parent_name =
if self._database.HasInterface(parent_name):
return res[1:]
def _merge_ext_attrs(self, old_attrs, new_attrs):
"""Merges two sets of extended attributes.
Returns: True if old_attrs has changed.
changed = False
for (name, value) in new_attrs.items():
if name in old_attrs and old_attrs[name] == value:
pass # Identical
if name == 'ImplementedAs' and name in old_attrs:
old_attrs[name] = value
changed = True
return changed
def _merge_nodes(self, old_list, new_list, import_options):
"""Merges two lists of nodes. Annotates nodes with the source of each
True if the old_list has changed.
old_list -- the list to merge into.
new_list -- list containing more nodes.
import_options -- controls how merging is done.
changed = False
source = import_options.source
old_signatures_map = self._build_signatures_map(old_list)
new_signatures_map = self._build_signatures_map(new_list)
# Merge new items
for (sig, new_node) in new_signatures_map.items():
if sig not in old_signatures_map:
# New node:
changed = True
# Merge old and new nodes:
old_node = old_signatures_map[sig]
if (source not in old_node.annotations
and source in new_node.annotations):
old_node.annotations[source] = new_node.annotations[source]
changed = True
# Maybe rename arguments:
if isinstance(old_node, IDLOperation):
for i in range(0, len(old_node.arguments)):
old_arg = old_node.arguments[i]
new_arg = new_node.arguments[i]
old_arg_name =
new_arg_name =
if (old_arg_name != new_arg_name
and (old_arg_name == 'arg'
or old_arg_name.endswith('Arg')
or import_options.rename_operation_arguments_on_merge)):
old_node.arguments[i].id = new_arg_name
changed = True
if self._merge_ext_attrs(old_arg.ext_attrs, new_arg.ext_attrs):
changed = True
# Maybe merge annotations:
if (isinstance(old_node, IDLAttribute) or
isinstance(old_node, IDLOperation)):
if self._merge_ext_attrs(old_node.ext_attrs, new_node.ext_attrs):
changed = True
# Remove annotations on obsolete items from the same source
if import_options.obsolete_old_declarations:
for (sig, old_node) in old_signatures_map.items():
if (source in old_node.annotations
and sig not in new_signatures_map):
_logger.warn('%s not available in %s anymore' %
(sig, source))
del old_node.annotations[source]
changed = True
return changed
def _merge_interfaces(self, old_interface, new_interface, import_options):
"""Merges the new_interface into the old_interface, annotating the
interface with the sources of each change."""
changed = False
source = import_options.source
if (source and source not in old_interface.annotations and
source in new_interface.annotations and
not new_interface.is_supplemental):
old_interface.annotations[source] = new_interface.annotations[source]
changed = True
def merge_list(what):
old_list = old_interface.__dict__[what]
new_list = new_interface.__dict__[what]
if what != 'parents' and !=
for node in new_list:
node.doc_js_interface_name =
node.ext_attrs['ImplementedBy'] =
changed = self._merge_nodes(old_list, new_list, import_options)
# Delete list items with zero remaining annotations.
if changed and import_options.obsolete_old_declarations:
def has_annotations(idl_node):
return len(idl_node.annotations)
old_interface.__dict__[what] = filter(has_annotations, old_list)
return changed
# Smartly merge various declarations:
if merge_list('parents'):
changed = True
if merge_list('constants'):
changed = True
if merge_list('attributes'):
changed = True
if merge_list('operations'):
changed = True
if self._merge_ext_attrs(old_interface.ext_attrs, new_interface.ext_attrs):
changed = True'merged interface %s (changed=%s, supplemental=%s)' %
(, changed, new_interface.is_supplemental))
return changed
def _merge_impl_stmt(self, impl_stmt, import_options):
"""Applies "X implements Y" statemetns on the proper places in the
implementor_name =
implemented_name ='merging impl stmt %s implements %s' %
(implementor_name, implemented_name))
source = import_options.source
if self._database.HasInterface(implementor_name):
interface = self._database.GetInterface(implementor_name)
if interface.parents is None:
interface.parents = []
for parent in interface.parents:
if == implemented_name:
if source and source not in parent.annotations:
parent.annotations[source] = IDLAnnotation(
# not found, so add new one
parent = IDLParentInterface(None)
parent.type = IDLType(implemented_name)
if source:
parent.annotations[source] = IDLAnnotation(
def merge_imported_interfaces(self):
"""Merges all imported interfaces and loads them into the DB."""
# Step 1: Pre process imported interfaces
for interface, import_options in self._imported_interfaces:
self._annotate(interface, import_options)
# Step 2: Add all new interfaces and merge overlapping ones
for interface, import_options in self._imported_interfaces:
if not interface.is_supplemental:
if self._database.HasInterface(
old_interface = self._database.GetInterface(
self._merge_interfaces(old_interface, interface, import_options)
if import_options.add_new_interfaces:
# Step 3: Merge in supplemental interfaces
for interface, import_options in self._imported_interfaces:
if interface.is_supplemental:
target_name = interface.ext_attrs['Supplemental']
if target_name:
# [Supplemental=Window] - merge into Window.
target = target_name
# [Supplemental] - merge into existing inteface with same name.
target =
if self._database.HasInterface(target):
old_interface = self._database.GetInterface(target)
self._merge_interfaces(old_interface, interface, import_options)
_logger.warning("Supplemental target '%s' not found", target)
# Step 4: Resolve 'implements' statements
for impl_stmt, import_options in self._impl_stmts:
self._merge_impl_stmt(impl_stmt, import_options)
self._impl_stmts = []
self._imported_interfaces = []
def import_idl_files(self, file_paths, import_options, parallel):
if parallel:
# Parse the IDL files in parallel.
pool = multiprocessing.Pool()
for file_path in file_paths:
[ file_path, import_options],
callback = lambda idl_file:
self._process_idl_file(idl_file, import_options))
# Parse the IDL files in serial.
for file_path in file_paths:
idl_file = _load_idl_file(file_path, import_options)
self._process_idl_file(idl_file, import_options)
def _process_idl_file(self, idl_file,
self._rename_types(idl_file, import_options)
def enabled(idl_node):
return self._is_node_enabled(idl_node, import_options.idl_defines)
for interface in idl_file.interfaces:
if not self._is_node_enabled(interface, import_options.idl_defines):'skipping interface %s (source=%s)'
% (, import_options.source))
continue'importing interface %s (source=%s file=%s)'
% (, import_options.source, idl_file))
interface.attributes = filter(enabled, interface.attributes)
interface.operations = filter(enabled, interface.operations)
self._imported_interfaces.append((interface, import_options))
for implStmt in idl_file.implementsStatements:
self._impl_stmts.append((implStmt, import_options))
for enum in idl_file.enums:
def _is_node_enabled(self, node, idl_defines):
if not 'Conditional' in node.ext_attrs:
return True
def enabled(condition):
return 'ENABLE_%s' % condition in idl_defines
conditional = node.ext_attrs['Conditional']
if conditional.find('&') != -1:
for condition in conditional.split('&'):
condition = condition.strip()
if not enabled(condition):
return False
return True
for condition in conditional.split('|'):
condition = condition.strip()
if enabled(condition):
return True
return False
def fix_displacements(self, source):
"""E.g. In W3C, something is declared on HTMLDocument but in WebKit
its on Document, so we need to mark that something in HTMLDocument
with @WebKit(via=Document). The 'via' attribute specifies the
parent interface that has the declaration."""
for interface in self._database.GetInterfaces():
changed = False'fixing displacements in %s' %
for parent_interface in self._get_parent_interfaces(interface):'scanning parent %s of %s' %
def fix_nodes(local_list, parent_list):
changed = False
parent_signatures_map = self._build_signatures_map(
for idl_node in local_list:
sig = self._sign(idl_node)
if sig in parent_signatures_map:
parent_member = parent_signatures_map[sig]
if (source in parent_member.annotations
and source not in idl_node.annotations
not in parent_member.annotations[source]):
idl_node.annotations[source] = IDLAnnotation(
changed = True
return changed
changed = fix_nodes(interface.constants,
parent_interface.constants) or changed
changed = fix_nodes(interface.attributes,
parent_interface.attributes) or changed
changed = fix_nodes(interface.operations,
parent_interface.operations) or changed
if changed:'fixed displaced declarations in %s' %
def normalize_annotations(self, sources):
"""Makes the IDLs less verbose by removing annotation attributes
that are identical to the ones defined at the interface level.
sources -- list of source names to normalize."""
for interface in self._database.GetInterfaces():
_logger.debug('normalizing annotations for %s' %
for source in sources:
if (source not in interface.annotations or
not interface.annotations[source]):
top_level_annotation = interface.annotations[source]
def normalize(idl_node):
if (source in idl_node.annotations
and idl_node.annotations[source]):
annotation = idl_node.annotations[source]
for name, value in annotation.items():
if (name in top_level_annotation
and value == top_level_annotation[name]):
del annotation[name]
map(normalize, interface.parents)
map(normalize, interface.constants)
map(normalize, interface.attributes)
map(normalize, interface.operations)
def fetch_constructor_data(self, options):
window_interface = self._database.GetInterface('Window')
for attr in window_interface.attributes:
type =
if not type.endswith('Constructor'):
type = re.sub('(Constructor)+$', '', type)
# TODO(antonm): Ideally we'd like to have pristine copy of WebKit IDLs and fetch
# this information directly from it. Unfortunately right now database is massaged
# a lot so it's difficult to maintain necessary information on Window itself.
interface = self._database.GetInterface(type)
if 'V8EnabledPerContext' in attr.ext_attrs:
interface.ext_attrs['synthesizedV8EnabledPerContext'] = \
if 'V8EnabledAtRuntime' in attr.ext_attrs:
interface.ext_attrs['synthesizedV8EnabledAtRuntime'] = \
attr.ext_attrs['V8EnabledAtRuntime'] or