summaryrefslogtreecommitdiff
path: root/utils/ipc/mojo/public/tools/mojom
diff options
context:
space:
mode:
authorPaul Elder <paul.elder@ideasonboard.com>2020-09-08 20:47:19 +0900
committerPaul Elder <paul.elder@ideasonboard.com>2021-05-26 13:03:27 +0900
commit139d8855747799da9218f36720004fb1927bd2ef (patch)
tree7f3f47e68e08cfcf19ee9f37cc91f31690b9fc53 /utils/ipc/mojo/public/tools/mojom
parenta7ded8e8f5dd0ae0841960280fe9a7107f945a34 (diff)
utils: ipc: Update mojo
Update mojo from the Chromium repository. The commit from which this was taken is: 9c138d992bfc1fb8f4f7bcf58d00bf19c219e4e2 "Updating trunk VERSION from 4523.0 to 4524.0" The update-mojo.sh script was used for this update. Bug: https://bugs.libcamera.org/show_bug.cgi?id=34 Signed-off-by: Paul Elder <paul.elder@ideasonboard.com> Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Diffstat (limited to 'utils/ipc/mojo/public/tools/mojom')
-rw-r--r--utils/ipc/mojo/public/tools/mojom/README.md2
-rwxr-xr-xutils/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py3
-rw-r--r--utils/ipc/mojo/public/tools/mojom/mojom/BUILD.gn1
-rw-r--r--utils/ipc/mojo/public/tools/mojom/mojom/generate/generator.py14
-rw-r--r--utils/ipc/mojo/public/tools/mojom/mojom/generate/module.py163
-rw-r--r--utils/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py11
-rw-r--r--utils/ipc/mojo/public/tools/mojom/mojom/generate/translate.py26
-rwxr-xr-xutils/ipc/mojo/public/tools/mojom/mojom_parser.py210
-rw-r--r--utils/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py46
9 files changed, 376 insertions, 100 deletions
diff --git a/utils/ipc/mojo/public/tools/mojom/README.md b/utils/ipc/mojo/public/tools/mojom/README.md
index 6a4ff78a..e5d17ab0 100644
--- a/utils/ipc/mojo/public/tools/mojom/README.md
+++ b/utils/ipc/mojo/public/tools/mojom/README.md
@@ -3,7 +3,7 @@
The Mojom format is an interface definition language (IDL) for describing
interprocess communication (IPC) messages and data types for use with the
low-level cross-platform
-[Mojo IPC library](https://chromium.googlesource.com/chromium/src/+/master/mojo/public/c/system/README.md).
+[Mojo IPC library](https://chromium.googlesource.com/chromium/src/+/main/mojo/public/c/system/README.md).
This directory consists of a `mojom` Python module, its tests, and supporting
command-line tools. The Python module implements the parser used by the
diff --git a/utils/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py b/utils/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py
index 7e746112..08bd672f 100755
--- a/utils/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py
+++ b/utils/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py
@@ -131,7 +131,8 @@ def _ValidateDelta(root, delta):
'renamed, please add a [RenamedFrom] attribute to the new type. This '
'can be deleted by a subsequent change.' % qualified_name)
- if not new_types[new_name].IsBackwardCompatible(kind):
+ checker = module.BackwardCompatibilityChecker()
+ if not checker.IsBackwardCompatible(new_types[new_name], kind):
raise Exception('Stable type %s appears to have changed in a way which '
'breaks backward-compatibility. Please fix!\n\nIf you '
'believe this assessment to be incorrect, please file a '
diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/BUILD.gn b/utils/ipc/mojo/public/tools/mojom/mojom/BUILD.gn
index 7416ef19..51facc0c 100644
--- a/utils/ipc/mojo/public/tools/mojom/mojom/BUILD.gn
+++ b/utils/ipc/mojo/public/tools/mojom/mojom/BUILD.gn
@@ -8,7 +8,6 @@ group("mojom") {
"error.py",
"fileutil.py",
"generate/__init__.py",
- "generate/constant_resolver.py",
"generate/generator.py",
"generate/module.py",
"generate/pack.py",
diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/generate/generator.py b/utils/ipc/mojo/public/tools/mojom/mojom/generate/generator.py
index de62260a..4a1c73fc 100644
--- a/utils/ipc/mojo/public/tools/mojom/mojom/generate/generator.py
+++ b/utils/ipc/mojo/public/tools/mojom/mojom/generate/generator.py
@@ -136,9 +136,14 @@ class Stylizer(object):
def WriteFile(contents, full_path):
# If |contents| is same with the file content, we skip updating.
+ if not isinstance(contents, bytes):
+ data = contents.encode('utf8')
+ else:
+ data = contents
+
if os.path.isfile(full_path):
with open(full_path, 'rb') as destination_file:
- if destination_file.read() == contents:
+ if destination_file.read() == data:
return
# Make sure the containing directory exists.
@@ -146,11 +151,8 @@ def WriteFile(contents, full_path):
fileutil.EnsureDirectoryExists(full_dir)
# Dump the data to disk.
- with open(full_path, "wb") as f:
- if not isinstance(contents, bytes):
- f.write(contents.encode('utf-8'))
- else:
- f.write(contents)
+ with open(full_path, 'wb') as f:
+ f.write(data)
def AddComputedData(module):
diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/generate/module.py b/utils/ipc/mojo/public/tools/mojom/mojom/generate/module.py
index 8547ff64..9bdb28e0 100644
--- a/utils/ipc/mojo/public/tools/mojom/mojom/generate/module.py
+++ b/utils/ipc/mojo/public/tools/mojom/mojom/generate/module.py
@@ -12,7 +12,33 @@
# method = interface.AddMethod('Tat', 0)
# method.AddParameter('baz', 0, mojom.INT32)
-import pickle
+import sys
+if sys.version_info.major == 2:
+ import cPickle as pickle
+else:
+ import pickle
+from uuid import UUID
+
+
+class BackwardCompatibilityChecker(object):
+ """Used for memoization while recursively checking two type definitions for
+ backward-compatibility."""
+
+ def __init__(self):
+ self._cache = {}
+
+ def IsBackwardCompatible(self, new_kind, old_kind):
+ key = (new_kind, old_kind)
+ result = self._cache.get(key)
+ if result is None:
+ # Assume they're compatible at first to effectively ignore recursive
+ # checks between these types, e.g. if both kinds are a struct or union
+ # that references itself in a field.
+ self._cache[key] = True
+ result = new_kind.IsBackwardCompatible(old_kind, self)
+ self._cache[key] = result
+ return result
+
# We use our own version of __repr__ when displaying the AST, as the
# AST currently doesn't capture which nodes are reference (e.g. to
@@ -114,6 +140,10 @@ class Kind(object):
# during a subsequent run of the parser.
return hash((self.spec, self.parent_kind))
+ # pylint: disable=unused-argument
+ def IsBackwardCompatible(self, rhs, checker):
+ return self == rhs
+
class ReferenceKind(Kind):
"""ReferenceKind represents pointer and handle types.
@@ -195,6 +225,10 @@ class ReferenceKind(Kind):
def __hash__(self):
return hash((super(ReferenceKind, self).__hash__(), self.is_nullable))
+ def IsBackwardCompatible(self, rhs, checker):
+ return (super(ReferenceKind, self).IsBackwardCompatible(rhs, checker)
+ and self.is_nullable == rhs.is_nullable)
+
# Initialize the set of primitive types. These can be accessed by clients.
BOOL = Kind('b')
@@ -253,9 +287,13 @@ PRIMITIVES = (
)
ATTRIBUTE_MIN_VERSION = 'MinVersion'
+ATTRIBUTE_DEFAULT = 'Default'
ATTRIBUTE_EXTENSIBLE = 'Extensible'
+ATTRIBUTE_NO_INTERRUPT = 'NoInterrupt'
ATTRIBUTE_STABLE = 'Stable'
ATTRIBUTE_SYNC = 'Sync'
+ATTRIBUTE_UNLIMITED_SIZE = 'UnlimitedSize'
+ATTRIBUTE_UUID = 'Uuid'
class NamedValue(object):
@@ -274,6 +312,9 @@ class NamedValue(object):
and (self.parent_kind, self.mojom_name) == (rhs.parent_kind,
rhs.mojom_name))
+ def __hash__(self):
+ return hash((self.parent_kind, self.mojom_name))
+
class BuiltinValue(object):
def __init__(self, value):
@@ -368,21 +409,19 @@ class Field(object):
class StructField(Field):
- pass
+ def __hash__(self):
+ return super(Field, self).__hash__()
class UnionField(Field):
pass
-def _IsFieldBackwardCompatible(new_field, old_field):
+def _IsFieldBackwardCompatible(new_field, old_field, checker):
if (new_field.min_version or 0) != (old_field.min_version or 0):
return False
- if isinstance(new_field.kind, (Enum, Struct, Union)):
- return new_field.kind.IsBackwardCompatible(old_field.kind)
-
- return new_field.kind == old_field.kind
+ return checker.IsBackwardCompatible(new_field.kind, old_field.kind)
class Struct(ReferenceKind):
@@ -457,7 +496,7 @@ class Struct(ReferenceKind):
for constant in self.constants:
constant.Stylize(stylizer)
- def IsBackwardCompatible(self, older_struct):
+ def IsBackwardCompatible(self, older_struct, checker):
"""This struct is backward-compatible with older_struct if and only if all
of the following conditions hold:
- Any newly added field is tagged with a [MinVersion] attribute specifying
@@ -496,7 +535,7 @@ class Struct(ReferenceKind):
old_field = old_fields[ordinal]
if (old_field.min_version or 0) > max_old_min_version:
max_old_min_version = old_field.min_version
- if not _IsFieldBackwardCompatible(new_field, old_field):
+ if not _IsFieldBackwardCompatible(new_field, old_field, checker):
# Type or min-version mismatch between old and new versions of the same
# ordinal field.
return False
@@ -590,7 +629,7 @@ class Union(ReferenceKind):
for field in self.fields:
field.Stylize(stylizer)
- def IsBackwardCompatible(self, older_union):
+ def IsBackwardCompatible(self, older_union, checker):
"""This union is backward-compatible with older_union if and only if all
of the following conditions hold:
- Any newly added field is tagged with a [MinVersion] attribute specifying
@@ -623,7 +662,7 @@ class Union(ReferenceKind):
if not new_field:
# A field was removed, which is not OK.
return False
- if not _IsFieldBackwardCompatible(new_field, old_field):
+ if not _IsFieldBackwardCompatible(new_field, old_field, checker):
# An field changed its type or MinVersion, which is not OK.
return False
old_min_version = old_field.min_version or 0
@@ -703,6 +742,10 @@ class Array(ReferenceKind):
def __hash__(self):
return id(self)
+ def IsBackwardCompatible(self, rhs, checker):
+ return (isinstance(rhs, Array) and self.length == rhs.length
+ and checker.IsBackwardCompatible(self.kind, rhs.kind))
+
class Map(ReferenceKind):
"""A map.
@@ -747,6 +790,11 @@ class Map(ReferenceKind):
def __hash__(self):
return id(self)
+ def IsBackwardCompatible(self, rhs, checker):
+ return (isinstance(rhs, Map)
+ and checker.IsBackwardCompatible(self.key_kind, rhs.key_kind)
+ and checker.IsBackwardCompatible(self.value_kind, rhs.value_kind))
+
class PendingRemote(ReferenceKind):
ReferenceKind.AddSharedProperty('kind')
@@ -768,6 +816,10 @@ class PendingRemote(ReferenceKind):
def __hash__(self):
return id(self)
+ def IsBackwardCompatible(self, rhs, checker):
+ return (isinstance(rhs, PendingRemote)
+ and checker.IsBackwardCompatible(self.kind, rhs.kind))
+
class PendingReceiver(ReferenceKind):
ReferenceKind.AddSharedProperty('kind')
@@ -789,6 +841,10 @@ class PendingReceiver(ReferenceKind):
def __hash__(self):
return id(self)
+ def IsBackwardCompatible(self, rhs, checker):
+ return isinstance(rhs, PendingReceiver) and checker.IsBackwardCompatible(
+ self.kind, rhs.kind)
+
class PendingAssociatedRemote(ReferenceKind):
ReferenceKind.AddSharedProperty('kind')
@@ -810,6 +866,11 @@ class PendingAssociatedRemote(ReferenceKind):
def __hash__(self):
return id(self)
+ def IsBackwardCompatible(self, rhs, checker):
+ return isinstance(rhs,
+ PendingAssociatedRemote) and checker.IsBackwardCompatible(
+ self.kind, rhs.kind)
+
class PendingAssociatedReceiver(ReferenceKind):
ReferenceKind.AddSharedProperty('kind')
@@ -831,6 +892,11 @@ class PendingAssociatedReceiver(ReferenceKind):
def __hash__(self):
return id(self)
+ def IsBackwardCompatible(self, rhs, checker):
+ return isinstance(
+ rhs, PendingAssociatedReceiver) and checker.IsBackwardCompatible(
+ self.kind, rhs.kind)
+
class InterfaceRequest(ReferenceKind):
ReferenceKind.AddSharedProperty('kind')
@@ -851,6 +917,10 @@ class InterfaceRequest(ReferenceKind):
def __hash__(self):
return id(self)
+ def IsBackwardCompatible(self, rhs, checker):
+ return isinstance(rhs, InterfaceRequest) and checker.IsBackwardCompatible(
+ self.kind, rhs.kind)
+
class AssociatedInterfaceRequest(ReferenceKind):
ReferenceKind.AddSharedProperty('kind')
@@ -873,6 +943,11 @@ class AssociatedInterfaceRequest(ReferenceKind):
def __hash__(self):
return id(self)
+ def IsBackwardCompatible(self, rhs, checker):
+ return isinstance(
+ rhs, AssociatedInterfaceRequest) and checker.IsBackwardCompatible(
+ self.kind, rhs.kind)
+
class Parameter(object):
def __init__(self,
@@ -976,6 +1051,16 @@ class Method(object):
return self.attributes.get(ATTRIBUTE_SYNC) \
if self.attributes else None
+ @property
+ def allow_interrupt(self):
+ return not self.attributes.get(ATTRIBUTE_NO_INTERRUPT) \
+ if self.attributes else True
+
+ @property
+ def unlimited_message_size(self):
+ return self.attributes.get(ATTRIBUTE_UNLIMITED_SIZE) \
+ if self.attributes else False
+
def __eq__(self, rhs):
return (isinstance(rhs, Method) and
(self.mojom_name, self.ordinal, self.parameters,
@@ -1029,7 +1114,7 @@ class Interface(ReferenceKind):
for constant in self.constants:
constant.Stylize(stylizer)
- def IsBackwardCompatible(self, older_interface):
+ def IsBackwardCompatible(self, older_interface, checker):
"""This interface is backward-compatible with older_interface if and only
if all of the following conditions hold:
- All defined methods in older_interface (when identified by ordinal) have
@@ -1067,8 +1152,8 @@ class Interface(ReferenceKind):
# A method was removed, which is not OK.
return False
- if not new_method.param_struct.IsBackwardCompatible(
- old_method.param_struct):
+ if not checker.IsBackwardCompatible(new_method.param_struct,
+ old_method.param_struct):
# The parameter list is not backward-compatible, which is not OK.
return False
@@ -1081,8 +1166,8 @@ class Interface(ReferenceKind):
if new_method.response_param_struct is None:
# A reply was removed from a message, which is not OK.
return False
- if not new_method.response_param_struct.IsBackwardCompatible(
- old_method.response_param_struct):
+ if not checker.IsBackwardCompatible(new_method.response_param_struct,
+ old_method.response_param_struct):
# The new message's reply is not backward-compatible with the old
# message's reply, which is not OK.
return False
@@ -1120,6 +1205,20 @@ class Interface(ReferenceKind):
self.attributes) == (rhs.mojom_name, rhs.methods, rhs.enums,
rhs.constants, rhs.attributes))
+ @property
+ def uuid(self):
+ uuid_str = self.attributes.get(ATTRIBUTE_UUID) if self.attributes else None
+ if uuid_str is None:
+ return None
+
+ try:
+ u = UUID(uuid_str)
+ except:
+ raise ValueError('Invalid format for Uuid attribute on interface {}. '
+ 'Expected standard RFC 4122 string representation of '
+ 'a UUID.'.format(self.mojom_name))
+ return (int(u.hex[:16], 16), int(u.hex[16:], 16))
+
def __hash__(self):
return id(self)
@@ -1144,6 +1243,11 @@ class AssociatedInterface(ReferenceKind):
def __hash__(self):
return id(self)
+ def IsBackwardCompatible(self, rhs, checker):
+ return isinstance(rhs,
+ AssociatedInterface) and checker.IsBackwardCompatible(
+ self.kind, rhs.kind)
+
class EnumField(object):
def __init__(self,
@@ -1161,6 +1265,11 @@ class EnumField(object):
self.name = stylizer.StylizeEnumField(self.mojom_name)
@property
+ def default(self):
+ return self.attributes.get(ATTRIBUTE_DEFAULT, False) \
+ if self.attributes else False
+
+ @property
def min_version(self):
return self.attributes.get(ATTRIBUTE_MIN_VERSION) \
if self.attributes else None
@@ -1186,6 +1295,7 @@ class Enum(Kind):
self.attributes = attributes
self.min_value = None
self.max_value = None
+ self.default_field = None
def Repr(self, as_ref=True):
if as_ref:
@@ -1216,7 +1326,8 @@ class Enum(Kind):
prefix = self.module.GetNamespacePrefix()
return '%s%s' % (prefix, self.mojom_name)
- def IsBackwardCompatible(self, older_enum):
+ # pylint: disable=unused-argument
+ def IsBackwardCompatible(self, older_enum, checker):
"""This enum is backward-compatible with older_enum if and only if one of
the following conditions holds:
- Neither enum is [Extensible] and both have the exact same set of valid
@@ -1250,9 +1361,10 @@ class Enum(Kind):
def __eq__(self, rhs):
return (isinstance(rhs, Enum) and
(self.mojom_name, self.native_only, self.fields, self.attributes,
- self.min_value,
- self.max_value) == (rhs.mojom_name, rhs.native_only, rhs.fields,
- rhs.attributes, rhs.min_value, rhs.max_value))
+ self.min_value, self.max_value,
+ self.default_field) == (rhs.mojom_name, rhs.native_only,
+ rhs.fields, rhs.attributes, rhs.min_value,
+ rhs.max_value, rhs.default_field))
def __hash__(self):
return id(self)
@@ -1272,6 +1384,7 @@ class Module(object):
self.attributes = attributes
self.imports = []
self.imported_kinds = {}
+ self.metadata = {}
def __repr__(self):
# Gives us a decent __repr__ for modules.
@@ -1285,6 +1398,9 @@ class Module(object):
rhs.imports, rhs.constants, rhs.enums,
rhs.structs, rhs.unions, rhs.interfaces))
+ def __hash__(self):
+ return id(self)
+
def Repr(self, as_ref=True):
if as_ref:
return '<%s path=%r mojom_namespace=%r>' % (
@@ -1555,6 +1671,13 @@ def HasSyncMethods(interface):
return False
+def HasUninterruptableMethods(interface):
+ for method in interface.methods:
+ if not method.allow_interrupt:
+ return True
+ return False
+
+
def ContainsHandlesOrInterfaces(kind):
"""Check if the kind contains any handles.
diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py b/utils/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py
index 7a300560..0da90058 100644
--- a/utils/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py
+++ b/utils/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py
@@ -75,9 +75,8 @@ def PrecompileTemplates(generator_modules, output_dir):
os.path.dirname(module.__file__), generator.GetTemplatePrefix())
]))
jinja_env.filters.update(generator.GetFilters())
- jinja_env.compile_templates(
- os.path.join(output_dir, "%s.zip" % generator.GetTemplatePrefix()),
- extensions=["tmpl"],
- zip="stored",
- py_compile=True,
- ignore_errors=False)
+ jinja_env.compile_templates(os.path.join(
+ output_dir, "%s.zip" % generator.GetTemplatePrefix()),
+ extensions=["tmpl"],
+ zip="stored",
+ ignore_errors=False)
diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/generate/translate.py b/utils/ipc/mojo/public/tools/mojom/mojom/generate/translate.py
index d6df3ca6..7580b780 100644
--- a/utils/ipc/mojo/public/tools/mojom/mojom/generate/translate.py
+++ b/utils/ipc/mojo/public/tools/mojom/mojom/generate/translate.py
@@ -472,6 +472,9 @@ def _Method(module, parsed_method, interface):
"attribute. If no response parameters are needed, you "
"could use an empty response parameter list, i.e., "
"\"=> ()\".")
+ # And only methods with the [Sync] attribute can specify [NoInterrupt].
+ if not method.allow_interrupt and not method.sync:
+ raise Exception("Only [Sync] methods can be marked [NoInterrupt].")
return method
@@ -592,6 +595,16 @@ def _Enum(module, parsed_enum, parent_kind):
map(lambda field: _EnumField(module, enum, field),
parsed_enum.enum_value_list))
_ResolveNumericEnumValues(enum)
+ # TODO(https://crbug.com/731893): Require a default value to be
+ # specified.
+ for field in enum.fields:
+ if field.default:
+ if not enum.extensible:
+ raise Exception('Non-extensible enums may not specify a default')
+ if enum.default_field is not None:
+ raise Exception(
+ 'Only one enumerator value may be specified as the default')
+ enum.default_field = field
module.kinds[enum.spec] = enum
@@ -650,7 +663,9 @@ def _CollectReferencedKinds(module, all_defined_kinds):
if mojom.IsMapKind(kind):
return (extract_referenced_user_kinds(kind.key_kind) +
extract_referenced_user_kinds(kind.value_kind))
- if mojom.IsInterfaceRequestKind(kind) or mojom.IsAssociatedKind(kind):
+ if (mojom.IsInterfaceRequestKind(kind) or mojom.IsAssociatedKind(kind)
+ or mojom.IsPendingRemoteKind(kind)
+ or mojom.IsPendingReceiverKind(kind)):
return [kind.kind]
if mojom.IsStructKind(kind):
return [kind]
@@ -678,12 +693,9 @@ def _CollectReferencedKinds(module, all_defined_kinds):
for method in interface.methods:
for param in itertools.chain(method.parameters or [],
method.response_parameters or []):
- if (mojom.IsStructKind(param.kind) or mojom.IsUnionKind(param.kind)
- or mojom.IsEnumKind(param.kind)
- or mojom.IsAnyInterfaceKind(param.kind)):
- for referenced_kind in extract_referenced_user_kinds(param.kind):
- sanitized_kind = sanitize_kind(referenced_kind)
- referenced_user_kinds[sanitized_kind.spec] = sanitized_kind
+ for referenced_kind in extract_referenced_user_kinds(param.kind):
+ sanitized_kind = sanitize_kind(referenced_kind)
+ referenced_user_kinds[sanitized_kind.spec] = sanitized_kind
return referenced_user_kinds
diff --git a/utils/ipc/mojo/public/tools/mojom/mojom_parser.py b/utils/ipc/mojo/public/tools/mojom/mojom_parser.py
index 12adbfb9..eb90c825 100755
--- a/utils/ipc/mojo/public/tools/mojom/mojom_parser.py
+++ b/utils/ipc/mojo/public/tools/mojom/mojom_parser.py
@@ -14,6 +14,8 @@ import argparse
import codecs
import errno
import json
+import logging
+import multiprocessing
import os
import os.path
import sys
@@ -25,6 +27,19 @@ from mojom.parse import parser
from mojom.parse import conditional_features
+# Disable this for easier debugging.
+# In Python 2, subprocesses just hang when exceptions are thrown :(.
+_ENABLE_MULTIPROCESSING = sys.version_info[0] > 2
+
+if sys.version_info < (3, 4):
+ _MULTIPROCESSING_USES_FORK = sys.platform.startswith('linux')
+else:
+ # https://docs.python.org/3/library/multiprocessing.html#:~:text=bpo-33725
+ if __name__ == '__main__' and sys.platform == 'darwin':
+ multiprocessing.set_start_method('fork')
+ _MULTIPROCESSING_USES_FORK = multiprocessing.get_start_method() == 'fork'
+
+
def _ResolveRelativeImportPath(path, roots):
"""Attempts to resolve a relative import path against a set of possible roots.
@@ -98,7 +113,7 @@ def _GetModuleFilename(mojom_filename):
def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts,
- dependencies, loaded_modules):
+ dependencies, loaded_modules, module_metadata):
"""Recursively ensures that a module and its dependencies are loaded.
Args:
@@ -111,10 +126,8 @@ def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts,
by absolute file path.
loaded_modules: A mapping of all modules loaded so far, including non-input
modules that were pulled in as transitive dependencies of the inputs.
- import_set: The working set of mojom imports processed so far in this
- call stack. Used to detect circular dependencies.
- import_stack: An ordered list of imports processed so far in this call
- stack. Used to report circular dependencies.
+ module_metadata: Metadata to be attached to every module loaded by this
+ helper.
Returns:
None
@@ -129,7 +142,7 @@ def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts,
for dep_abspath, dep_path in dependencies[mojom_abspath]:
if dep_abspath not in loaded_modules:
_EnsureInputLoaded(dep_abspath, dep_path, abs_paths, asts, dependencies,
- loaded_modules)
+ loaded_modules, module_metadata)
imports = {}
for imp in asts[mojom_abspath].import_list:
@@ -137,6 +150,7 @@ def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts,
imports[path] = loaded_modules[abs_paths[path]]
loaded_modules[mojom_abspath] = translate.OrderedModule(
asts[mojom_abspath], module_path, imports)
+ loaded_modules[mojom_abspath].metadata = dict(module_metadata)
def _CollectAllowedImportsFromBuildMetadata(build_metadata_filename):
@@ -157,10 +171,67 @@ def _CollectAllowedImportsFromBuildMetadata(build_metadata_filename):
return allowed_imports
+# multiprocessing helper.
+def _ParseAstHelper(args):
+ mojom_abspath, enabled_features = args
+ with codecs.open(mojom_abspath, encoding='utf-8') as f:
+ ast = parser.Parse(f.read(), mojom_abspath)
+ conditional_features.RemoveDisabledDefinitions(ast, enabled_features)
+ return mojom_abspath, ast
+
+
+# multiprocessing helper.
+def _SerializeHelper(args):
+ mojom_abspath, mojom_path = args
+ module_path = os.path.join(_SerializeHelper.output_root_path,
+ _GetModuleFilename(mojom_path))
+ module_dir = os.path.dirname(module_path)
+ if not os.path.exists(module_dir):
+ try:
+ # Python 2 doesn't support exist_ok on makedirs(), so we just ignore
+ # that failure if it happens. It's possible during build due to races
+ # among build steps with module outputs in the same directory.
+ os.makedirs(module_dir)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ with open(module_path, 'wb') as f:
+ _SerializeHelper.loaded_modules[mojom_abspath].Dump(f)
+
+
+def _Shard(target_func, args, processes=None):
+ args = list(args)
+ if processes is None:
+ processes = multiprocessing.cpu_count()
+ # Seems optimal to have each process perform at least 2 tasks.
+ processes = min(processes, len(args) // 2)
+
+ if sys.platform == 'win32':
+ # TODO(crbug.com/1190269) - we can't use more than 56
+ # cores on Windows or Python3 may hang.
+ processes = min(processes, 56)
+
+ # Don't spin up processes unless there is enough work to merit doing so.
+ if not _ENABLE_MULTIPROCESSING or processes < 2:
+ for result in map(target_func, args):
+ yield result
+ return
+
+ pool = multiprocessing.Pool(processes=processes)
+ try:
+ for result in pool.imap_unordered(target_func, args):
+ yield result
+ finally:
+ pool.close()
+ pool.join() # Needed on Windows to avoid WindowsError during terminate.
+ pool.terminate()
+
+
def _ParseMojoms(mojom_files,
input_root_paths,
output_root_path,
enabled_features,
+ module_metadata,
allowed_imports=None):
"""Parses a set of mojom files and produces serialized module outputs.
@@ -176,6 +247,8 @@ def _ParseMojoms(mojom_files,
modules for any transitive dependencies not listed in mojom_files.
enabled_features: A list of enabled feature names, controlling which AST
nodes are filtered by [EnableIf] attributes.
+ module_metadata: A list of 2-tuples representing metadata key-value pairs to
+ attach to each compiled module output.
Returns:
None.
@@ -193,72 +266,79 @@ def _ParseMojoms(mojom_files,
for abs_path in mojom_files)
abs_paths = dict(
(path, abs_path) for abs_path, path in mojom_files_to_parse.items())
- for mojom_abspath, _ in mojom_files_to_parse.items():
- with codecs.open(mojom_abspath, encoding='utf-8') as f:
- ast = parser.Parse(''.join(f.readlines()), mojom_abspath)
- conditional_features.RemoveDisabledDefinitions(ast, enabled_features)
- loaded_mojom_asts[mojom_abspath] = ast
- invalid_imports = []
- for imp in ast.import_list:
- import_abspath = _ResolveRelativeImportPath(imp.import_filename,
- input_root_paths)
- if allowed_imports and import_abspath not in allowed_imports:
- invalid_imports.append(imp.import_filename)
-
- abs_paths[imp.import_filename] = import_abspath
- if import_abspath in mojom_files_to_parse:
- # This import is in the input list, so we're going to translate it
- # into a module below; however it's also a dependency of another input
- # module. We retain record of dependencies to help with input
- # processing later.
- input_dependencies[mojom_abspath].add((import_abspath,
- imp.import_filename))
- else:
- # We have an import that isn't being parsed right now. It must already
- # be parsed and have a module file sitting in a corresponding output
- # location.
- module_path = _GetModuleFilename(imp.import_filename)
- module_abspath = _ResolveRelativeImportPath(module_path,
- [output_root_path])
- with open(module_abspath, 'rb') as module_file:
- loaded_modules[import_abspath] = module.Module.Load(module_file)
-
- if invalid_imports:
- raise ValueError(
- '\nThe file %s imports the following files not allowed by build '
- 'dependencies:\n\n%s\n' % (mojom_abspath,
- '\n'.join(invalid_imports)))
+ logging.info('Parsing %d .mojom into ASTs', len(mojom_files_to_parse))
+ map_args = ((mojom_abspath, enabled_features)
+ for mojom_abspath in mojom_files_to_parse)
+ for mojom_abspath, ast in _Shard(_ParseAstHelper, map_args):
+ loaded_mojom_asts[mojom_abspath] = ast
+
+ logging.info('Processing dependencies')
+ for mojom_abspath, ast in loaded_mojom_asts.items():
+ invalid_imports = []
+ for imp in ast.import_list:
+ import_abspath = _ResolveRelativeImportPath(imp.import_filename,
+ input_root_paths)
+ if allowed_imports and import_abspath not in allowed_imports:
+ invalid_imports.append(imp.import_filename)
+
+ abs_paths[imp.import_filename] = import_abspath
+ if import_abspath in mojom_files_to_parse:
+ # This import is in the input list, so we're going to translate it
+ # into a module below; however it's also a dependency of another input
+ # module. We retain record of dependencies to help with input
+ # processing later.
+ input_dependencies[mojom_abspath].add(
+ (import_abspath, imp.import_filename))
+ elif import_abspath not in loaded_modules:
+ # We have an import that isn't being parsed right now. It must already
+ # be parsed and have a module file sitting in a corresponding output
+ # location.
+ module_path = _GetModuleFilename(imp.import_filename)
+ module_abspath = _ResolveRelativeImportPath(module_path,
+ [output_root_path])
+ with open(module_abspath, 'rb') as module_file:
+ loaded_modules[import_abspath] = module.Module.Load(module_file)
+
+ if invalid_imports:
+ raise ValueError(
+ '\nThe file %s imports the following files not allowed by build '
+ 'dependencies:\n\n%s\n' % (mojom_abspath, '\n'.join(invalid_imports)))
+ logging.info('Loaded %d modules from dependencies', len(loaded_modules))
# At this point all transitive imports not listed as inputs have been loaded
# and we have a complete dependency tree of the unprocessed inputs. Now we can
# load all the inputs, resolving dependencies among them recursively as we go.
+ logging.info('Ensuring inputs are loaded')
num_existing_modules_loaded = len(loaded_modules)
for mojom_abspath, mojom_path in mojom_files_to_parse.items():
_EnsureInputLoaded(mojom_abspath, mojom_path, abs_paths, loaded_mojom_asts,
- input_dependencies, loaded_modules)
+ input_dependencies, loaded_modules, module_metadata)
assert (num_existing_modules_loaded +
len(mojom_files_to_parse) == len(loaded_modules))
# Now we have fully translated modules for every input and every transitive
# dependency. We can dump the modules to disk for other tools to use.
- for mojom_abspath, mojom_path in mojom_files_to_parse.items():
- module_path = os.path.join(output_root_path, _GetModuleFilename(mojom_path))
- module_dir = os.path.dirname(module_path)
- if not os.path.exists(module_dir):
- try:
- # Python 2 doesn't support exist_ok on makedirs(), so we just ignore
- # that failure if it happens. It's possible during build due to races
- # among build steps with module outputs in the same directory.
- os.makedirs(module_dir)
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise
- with open(module_path, 'wb') as f:
- loaded_modules[mojom_abspath].Dump(f)
+ logging.info('Serializing %d modules', len(mojom_files_to_parse))
+
+ # Windows does not use fork() for multiprocessing, so we'd need to pass
+ # loaded_module via IPC rather than via globals. Doing so is slower than not
+ # using multiprocessing.
+ _SerializeHelper.loaded_modules = loaded_modules
+ _SerializeHelper.output_root_path = output_root_path
+ # Doesn't seem to help past 4. Perhaps IO bound here?
+ processes = 4 if _MULTIPROCESSING_USES_FORK else 0
+ map_args = mojom_files_to_parse.items()
+ for _ in _Shard(_SerializeHelper, map_args, processes=processes):
+ pass
def Run(command_line):
+ debug_logging = os.environ.get('MOJOM_PARSER_DEBUG', '0') != '0'
+ logging.basicConfig(level=logging.DEBUG if debug_logging else logging.WARNING,
+ format='%(levelname).1s %(relativeCreated)6d %(message)s')
+ logging.info('Started (%s)', os.path.basename(sys.argv[0]))
+
arg_parser = argparse.ArgumentParser(
description="""
Parses one or more mojom files and produces corresponding module outputs fully
@@ -333,6 +413,16 @@ already present in the provided output root.""")
'build-time dependency checking for mojom imports, where each build '
'metadata file corresponds to a build target in the dependency graph of '
'a typical build system.')
+ arg_parser.add_argument(
+ '--add-module-metadata',
+ dest='module_metadata',
+ default=[],
+ action='append',
+ metavar='KEY=VALUE',
+ help='Adds a metadata key-value pair to the output module. This can be '
+ 'used by build toolchains to augment parsed mojom modules with product-'
+ 'specific metadata for later extraction and use by custom bindings '
+ 'generators.')
args, _ = arg_parser.parse_known_args(command_line)
if args.mojom_file_list:
@@ -353,8 +443,14 @@ already present in the provided output root.""")
else:
allowed_imports = None
+ module_metadata = list(
+ map(lambda kvp: tuple(kvp.split('=')), args.module_metadata))
_ParseMojoms(mojom_files, input_roots, output_root, args.enabled_features,
- allowed_imports)
+ module_metadata, allowed_imports)
+ logging.info('Finished')
+ # Exit without running GC, which can save multiple seconds due the large
+ # number of object created.
+ os._exit(0)
if __name__ == '__main__':
diff --git a/utils/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py b/utils/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py
index a0ee150e..65db4dc9 100644
--- a/utils/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py
+++ b/utils/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py
@@ -2,6 +2,7 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+from mojom.generate import module
from mojom_parser_test_case import MojomParserTestCase
@@ -20,9 +21,11 @@ class VersionCompatibilityTest(MojomParserTestCase):
self.assertEqual(set(old.keys()), set(new.keys()),
'Old and new test mojoms should use the same type names.')
+ checker = module.BackwardCompatibilityChecker()
compatibility_map = {}
for name in old.keys():
- compatibility_map[name] = new[name].IsBackwardCompatible(old[name])
+ compatibility_map[name] = checker.IsBackwardCompatible(
+ new[name], old[name])
return compatibility_map
def assertBackwardCompatible(self, old_mojom, new_mojom):
@@ -234,6 +237,47 @@ class VersionCompatibilityTest(MojomParserTestCase):
self.assertNotBackwardCompatible('union U { string a; };',
'union U { string? a; };')
+ def testFieldNestedTypeChanged(self):
+ """Changing the definition of a nested type within a field (such as an array
+ element or interface endpoint type) should only break backward-compatibility
+ if the changes to that type are not backward-compatible."""
+ self.assertBackwardCompatible(
+ """\
+ struct S { string a; };
+ struct T { array<S> ss; };
+ """, """\
+ struct S {
+ string a;
+ [MinVersion=1] string? b;
+ };
+ struct T { array<S> ss; };
+ """)
+ self.assertBackwardCompatible(
+ """\
+ interface F { Do(); };
+ struct S { pending_receiver<F> r; };
+ """, """\
+ interface F {
+ Do();
+ [MinVersion=1] Say();
+ };
+ struct S { pending_receiver<F> r; };
+ """)
+
+ def testRecursiveTypeChange(self):
+ """Recursive types do not break the compatibility checker."""
+ self.assertBackwardCompatible(
+ """\
+ struct S {
+ string a;
+ array<S> others;
+ };""", """\
+ struct S {
+ string a;
+ array<S> others;
+ [MinVersion=1] string? b;
+ };""")
+
def testUnionFieldBecomingNonOptional(self):
"""Changing a field from optional to non-optional breaks
backward-compatibility."""