diff options
Diffstat (limited to 'utils/ipc/mojo/public/tools/mojom')
9 files changed, 376 insertions, 100 deletions
diff --git a/utils/ipc/mojo/public/tools/mojom/README.md b/utils/ipc/mojo/public/tools/mojom/README.md index 6a4ff78a..e5d17ab0 100644 --- a/utils/ipc/mojo/public/tools/mojom/README.md +++ b/utils/ipc/mojo/public/tools/mojom/README.md @@ -3,7 +3,7 @@ The Mojom format is an interface definition language (IDL) for describing interprocess communication (IPC) messages and data types for use with the low-level cross-platform -[Mojo IPC library](https://chromium.googlesource.com/chromium/src/+/master/mojo/public/c/system/README.md). +[Mojo IPC library](https://chromium.googlesource.com/chromium/src/+/main/mojo/public/c/system/README.md). This directory consists of a `mojom` Python module, its tests, and supporting command-line tools. The Python module implements the parser used by the diff --git a/utils/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py b/utils/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py index 7e746112..08bd672f 100755 --- a/utils/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py +++ b/utils/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py @@ -131,7 +131,8 @@ def _ValidateDelta(root, delta): 'renamed, please add a [RenamedFrom] attribute to the new type. This ' 'can be deleted by a subsequent change.' % qualified_name) - if not new_types[new_name].IsBackwardCompatible(kind): + checker = module.BackwardCompatibilityChecker() + if not checker.IsBackwardCompatible(new_types[new_name], kind): raise Exception('Stable type %s appears to have changed in a way which ' 'breaks backward-compatibility. Please fix!\n\nIf you ' 'believe this assessment to be incorrect, please file a ' diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/BUILD.gn b/utils/ipc/mojo/public/tools/mojom/mojom/BUILD.gn index 7416ef19..51facc0c 100644 --- a/utils/ipc/mojo/public/tools/mojom/mojom/BUILD.gn +++ b/utils/ipc/mojo/public/tools/mojom/mojom/BUILD.gn @@ -8,7 +8,6 @@ group("mojom") { "error.py", "fileutil.py", "generate/__init__.py", - "generate/constant_resolver.py", "generate/generator.py", "generate/module.py", "generate/pack.py", diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/generate/generator.py b/utils/ipc/mojo/public/tools/mojom/mojom/generate/generator.py index de62260a..4a1c73fc 100644 --- a/utils/ipc/mojo/public/tools/mojom/mojom/generate/generator.py +++ b/utils/ipc/mojo/public/tools/mojom/mojom/generate/generator.py @@ -136,9 +136,14 @@ class Stylizer(object): def WriteFile(contents, full_path): # If |contents| is same with the file content, we skip updating. + if not isinstance(contents, bytes): + data = contents.encode('utf8') + else: + data = contents + if os.path.isfile(full_path): with open(full_path, 'rb') as destination_file: - if destination_file.read() == contents: + if destination_file.read() == data: return # Make sure the containing directory exists. @@ -146,11 +151,8 @@ def WriteFile(contents, full_path): fileutil.EnsureDirectoryExists(full_dir) # Dump the data to disk. - with open(full_path, "wb") as f: - if not isinstance(contents, bytes): - f.write(contents.encode('utf-8')) - else: - f.write(contents) + with open(full_path, 'wb') as f: + f.write(data) def AddComputedData(module): diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/generate/module.py b/utils/ipc/mojo/public/tools/mojom/mojom/generate/module.py index 8547ff64..9bdb28e0 100644 --- a/utils/ipc/mojo/public/tools/mojom/mojom/generate/module.py +++ b/utils/ipc/mojo/public/tools/mojom/mojom/generate/module.py @@ -12,7 +12,33 @@ # method = interface.AddMethod('Tat', 0) # method.AddParameter('baz', 0, mojom.INT32) -import pickle +import sys +if sys.version_info.major == 2: + import cPickle as pickle +else: + import pickle +from uuid import UUID + + +class BackwardCompatibilityChecker(object): + """Used for memoization while recursively checking two type definitions for + backward-compatibility.""" + + def __init__(self): + self._cache = {} + + def IsBackwardCompatible(self, new_kind, old_kind): + key = (new_kind, old_kind) + result = self._cache.get(key) + if result is None: + # Assume they're compatible at first to effectively ignore recursive + # checks between these types, e.g. if both kinds are a struct or union + # that references itself in a field. + self._cache[key] = True + result = new_kind.IsBackwardCompatible(old_kind, self) + self._cache[key] = result + return result + # We use our own version of __repr__ when displaying the AST, as the # AST currently doesn't capture which nodes are reference (e.g. to @@ -114,6 +140,10 @@ class Kind(object): # during a subsequent run of the parser. return hash((self.spec, self.parent_kind)) + # pylint: disable=unused-argument + def IsBackwardCompatible(self, rhs, checker): + return self == rhs + class ReferenceKind(Kind): """ReferenceKind represents pointer and handle types. @@ -195,6 +225,10 @@ class ReferenceKind(Kind): def __hash__(self): return hash((super(ReferenceKind, self).__hash__(), self.is_nullable)) + def IsBackwardCompatible(self, rhs, checker): + return (super(ReferenceKind, self).IsBackwardCompatible(rhs, checker) + and self.is_nullable == rhs.is_nullable) + # Initialize the set of primitive types. These can be accessed by clients. BOOL = Kind('b') @@ -253,9 +287,13 @@ PRIMITIVES = ( ) ATTRIBUTE_MIN_VERSION = 'MinVersion' +ATTRIBUTE_DEFAULT = 'Default' ATTRIBUTE_EXTENSIBLE = 'Extensible' +ATTRIBUTE_NO_INTERRUPT = 'NoInterrupt' ATTRIBUTE_STABLE = 'Stable' ATTRIBUTE_SYNC = 'Sync' +ATTRIBUTE_UNLIMITED_SIZE = 'UnlimitedSize' +ATTRIBUTE_UUID = 'Uuid' class NamedValue(object): @@ -274,6 +312,9 @@ class NamedValue(object): and (self.parent_kind, self.mojom_name) == (rhs.parent_kind, rhs.mojom_name)) + def __hash__(self): + return hash((self.parent_kind, self.mojom_name)) + class BuiltinValue(object): def __init__(self, value): @@ -368,21 +409,19 @@ class Field(object): class StructField(Field): - pass + def __hash__(self): + return super(Field, self).__hash__() class UnionField(Field): pass -def _IsFieldBackwardCompatible(new_field, old_field): +def _IsFieldBackwardCompatible(new_field, old_field, checker): if (new_field.min_version or 0) != (old_field.min_version or 0): return False - if isinstance(new_field.kind, (Enum, Struct, Union)): - return new_field.kind.IsBackwardCompatible(old_field.kind) - - return new_field.kind == old_field.kind + return checker.IsBackwardCompatible(new_field.kind, old_field.kind) class Struct(ReferenceKind): @@ -457,7 +496,7 @@ class Struct(ReferenceKind): for constant in self.constants: constant.Stylize(stylizer) - def IsBackwardCompatible(self, older_struct): + def IsBackwardCompatible(self, older_struct, checker): """This struct is backward-compatible with older_struct if and only if all of the following conditions hold: - Any newly added field is tagged with a [MinVersion] attribute specifying @@ -496,7 +535,7 @@ class Struct(ReferenceKind): old_field = old_fields[ordinal] if (old_field.min_version or 0) > max_old_min_version: max_old_min_version = old_field.min_version - if not _IsFieldBackwardCompatible(new_field, old_field): + if not _IsFieldBackwardCompatible(new_field, old_field, checker): # Type or min-version mismatch between old and new versions of the same # ordinal field. return False @@ -590,7 +629,7 @@ class Union(ReferenceKind): for field in self.fields: field.Stylize(stylizer) - def IsBackwardCompatible(self, older_union): + def IsBackwardCompatible(self, older_union, checker): """This union is backward-compatible with older_union if and only if all of the following conditions hold: - Any newly added field is tagged with a [MinVersion] attribute specifying @@ -623,7 +662,7 @@ class Union(ReferenceKind): if not new_field: # A field was removed, which is not OK. return False - if not _IsFieldBackwardCompatible(new_field, old_field): + if not _IsFieldBackwardCompatible(new_field, old_field, checker): # An field changed its type or MinVersion, which is not OK. return False old_min_version = old_field.min_version or 0 @@ -703,6 +742,10 @@ class Array(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return (isinstance(rhs, Array) and self.length == rhs.length + and checker.IsBackwardCompatible(self.kind, rhs.kind)) + class Map(ReferenceKind): """A map. @@ -747,6 +790,11 @@ class Map(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return (isinstance(rhs, Map) + and checker.IsBackwardCompatible(self.key_kind, rhs.key_kind) + and checker.IsBackwardCompatible(self.value_kind, rhs.value_kind)) + class PendingRemote(ReferenceKind): ReferenceKind.AddSharedProperty('kind') @@ -768,6 +816,10 @@ class PendingRemote(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return (isinstance(rhs, PendingRemote) + and checker.IsBackwardCompatible(self.kind, rhs.kind)) + class PendingReceiver(ReferenceKind): ReferenceKind.AddSharedProperty('kind') @@ -789,6 +841,10 @@ class PendingReceiver(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return isinstance(rhs, PendingReceiver) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + class PendingAssociatedRemote(ReferenceKind): ReferenceKind.AddSharedProperty('kind') @@ -810,6 +866,11 @@ class PendingAssociatedRemote(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return isinstance(rhs, + PendingAssociatedRemote) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + class PendingAssociatedReceiver(ReferenceKind): ReferenceKind.AddSharedProperty('kind') @@ -831,6 +892,11 @@ class PendingAssociatedReceiver(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return isinstance( + rhs, PendingAssociatedReceiver) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + class InterfaceRequest(ReferenceKind): ReferenceKind.AddSharedProperty('kind') @@ -851,6 +917,10 @@ class InterfaceRequest(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return isinstance(rhs, InterfaceRequest) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + class AssociatedInterfaceRequest(ReferenceKind): ReferenceKind.AddSharedProperty('kind') @@ -873,6 +943,11 @@ class AssociatedInterfaceRequest(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return isinstance( + rhs, AssociatedInterfaceRequest) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + class Parameter(object): def __init__(self, @@ -976,6 +1051,16 @@ class Method(object): return self.attributes.get(ATTRIBUTE_SYNC) \ if self.attributes else None + @property + def allow_interrupt(self): + return not self.attributes.get(ATTRIBUTE_NO_INTERRUPT) \ + if self.attributes else True + + @property + def unlimited_message_size(self): + return self.attributes.get(ATTRIBUTE_UNLIMITED_SIZE) \ + if self.attributes else False + def __eq__(self, rhs): return (isinstance(rhs, Method) and (self.mojom_name, self.ordinal, self.parameters, @@ -1029,7 +1114,7 @@ class Interface(ReferenceKind): for constant in self.constants: constant.Stylize(stylizer) - def IsBackwardCompatible(self, older_interface): + def IsBackwardCompatible(self, older_interface, checker): """This interface is backward-compatible with older_interface if and only if all of the following conditions hold: - All defined methods in older_interface (when identified by ordinal) have @@ -1067,8 +1152,8 @@ class Interface(ReferenceKind): # A method was removed, which is not OK. return False - if not new_method.param_struct.IsBackwardCompatible( - old_method.param_struct): + if not checker.IsBackwardCompatible(new_method.param_struct, + old_method.param_struct): # The parameter list is not backward-compatible, which is not OK. return False @@ -1081,8 +1166,8 @@ class Interface(ReferenceKind): if new_method.response_param_struct is None: # A reply was removed from a message, which is not OK. return False - if not new_method.response_param_struct.IsBackwardCompatible( - old_method.response_param_struct): + if not checker.IsBackwardCompatible(new_method.response_param_struct, + old_method.response_param_struct): # The new message's reply is not backward-compatible with the old # message's reply, which is not OK. return False @@ -1120,6 +1205,20 @@ class Interface(ReferenceKind): self.attributes) == (rhs.mojom_name, rhs.methods, rhs.enums, rhs.constants, rhs.attributes)) + @property + def uuid(self): + uuid_str = self.attributes.get(ATTRIBUTE_UUID) if self.attributes else None + if uuid_str is None: + return None + + try: + u = UUID(uuid_str) + except: + raise ValueError('Invalid format for Uuid attribute on interface {}. ' + 'Expected standard RFC 4122 string representation of ' + 'a UUID.'.format(self.mojom_name)) + return (int(u.hex[:16], 16), int(u.hex[16:], 16)) + def __hash__(self): return id(self) @@ -1144,6 +1243,11 @@ class AssociatedInterface(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return isinstance(rhs, + AssociatedInterface) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + class EnumField(object): def __init__(self, @@ -1161,6 +1265,11 @@ class EnumField(object): self.name = stylizer.StylizeEnumField(self.mojom_name) @property + def default(self): + return self.attributes.get(ATTRIBUTE_DEFAULT, False) \ + if self.attributes else False + + @property def min_version(self): return self.attributes.get(ATTRIBUTE_MIN_VERSION) \ if self.attributes else None @@ -1186,6 +1295,7 @@ class Enum(Kind): self.attributes = attributes self.min_value = None self.max_value = None + self.default_field = None def Repr(self, as_ref=True): if as_ref: @@ -1216,7 +1326,8 @@ class Enum(Kind): prefix = self.module.GetNamespacePrefix() return '%s%s' % (prefix, self.mojom_name) - def IsBackwardCompatible(self, older_enum): + # pylint: disable=unused-argument + def IsBackwardCompatible(self, older_enum, checker): """This enum is backward-compatible with older_enum if and only if one of the following conditions holds: - Neither enum is [Extensible] and both have the exact same set of valid @@ -1250,9 +1361,10 @@ class Enum(Kind): def __eq__(self, rhs): return (isinstance(rhs, Enum) and (self.mojom_name, self.native_only, self.fields, self.attributes, - self.min_value, - self.max_value) == (rhs.mojom_name, rhs.native_only, rhs.fields, - rhs.attributes, rhs.min_value, rhs.max_value)) + self.min_value, self.max_value, + self.default_field) == (rhs.mojom_name, rhs.native_only, + rhs.fields, rhs.attributes, rhs.min_value, + rhs.max_value, rhs.default_field)) def __hash__(self): return id(self) @@ -1272,6 +1384,7 @@ class Module(object): self.attributes = attributes self.imports = [] self.imported_kinds = {} + self.metadata = {} def __repr__(self): # Gives us a decent __repr__ for modules. @@ -1285,6 +1398,9 @@ class Module(object): rhs.imports, rhs.constants, rhs.enums, rhs.structs, rhs.unions, rhs.interfaces)) + def __hash__(self): + return id(self) + def Repr(self, as_ref=True): if as_ref: return '<%s path=%r mojom_namespace=%r>' % ( @@ -1555,6 +1671,13 @@ def HasSyncMethods(interface): return False +def HasUninterruptableMethods(interface): + for method in interface.methods: + if not method.allow_interrupt: + return True + return False + + def ContainsHandlesOrInterfaces(kind): """Check if the kind contains any handles. diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py b/utils/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py index 7a300560..0da90058 100644 --- a/utils/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py +++ b/utils/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py @@ -75,9 +75,8 @@ def PrecompileTemplates(generator_modules, output_dir): os.path.dirname(module.__file__), generator.GetTemplatePrefix()) ])) jinja_env.filters.update(generator.GetFilters()) - jinja_env.compile_templates( - os.path.join(output_dir, "%s.zip" % generator.GetTemplatePrefix()), - extensions=["tmpl"], - zip="stored", - py_compile=True, - ignore_errors=False) + jinja_env.compile_templates(os.path.join( + output_dir, "%s.zip" % generator.GetTemplatePrefix()), + extensions=["tmpl"], + zip="stored", + ignore_errors=False) diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/generate/translate.py b/utils/ipc/mojo/public/tools/mojom/mojom/generate/translate.py index d6df3ca6..7580b780 100644 --- a/utils/ipc/mojo/public/tools/mojom/mojom/generate/translate.py +++ b/utils/ipc/mojo/public/tools/mojom/mojom/generate/translate.py @@ -472,6 +472,9 @@ def _Method(module, parsed_method, interface): "attribute. If no response parameters are needed, you " "could use an empty response parameter list, i.e., " "\"=> ()\".") + # And only methods with the [Sync] attribute can specify [NoInterrupt]. + if not method.allow_interrupt and not method.sync: + raise Exception("Only [Sync] methods can be marked [NoInterrupt].") return method @@ -592,6 +595,16 @@ def _Enum(module, parsed_enum, parent_kind): map(lambda field: _EnumField(module, enum, field), parsed_enum.enum_value_list)) _ResolveNumericEnumValues(enum) + # TODO(https://crbug.com/731893): Require a default value to be + # specified. + for field in enum.fields: + if field.default: + if not enum.extensible: + raise Exception('Non-extensible enums may not specify a default') + if enum.default_field is not None: + raise Exception( + 'Only one enumerator value may be specified as the default') + enum.default_field = field module.kinds[enum.spec] = enum @@ -650,7 +663,9 @@ def _CollectReferencedKinds(module, all_defined_kinds): if mojom.IsMapKind(kind): return (extract_referenced_user_kinds(kind.key_kind) + extract_referenced_user_kinds(kind.value_kind)) - if mojom.IsInterfaceRequestKind(kind) or mojom.IsAssociatedKind(kind): + if (mojom.IsInterfaceRequestKind(kind) or mojom.IsAssociatedKind(kind) + or mojom.IsPendingRemoteKind(kind) + or mojom.IsPendingReceiverKind(kind)): return [kind.kind] if mojom.IsStructKind(kind): return [kind] @@ -678,12 +693,9 @@ def _CollectReferencedKinds(module, all_defined_kinds): for method in interface.methods: for param in itertools.chain(method.parameters or [], method.response_parameters or []): - if (mojom.IsStructKind(param.kind) or mojom.IsUnionKind(param.kind) - or mojom.IsEnumKind(param.kind) - or mojom.IsAnyInterfaceKind(param.kind)): - for referenced_kind in extract_referenced_user_kinds(param.kind): - sanitized_kind = sanitize_kind(referenced_kind) - referenced_user_kinds[sanitized_kind.spec] = sanitized_kind + for referenced_kind in extract_referenced_user_kinds(param.kind): + sanitized_kind = sanitize_kind(referenced_kind) + referenced_user_kinds[sanitized_kind.spec] = sanitized_kind return referenced_user_kinds diff --git a/utils/ipc/mojo/public/tools/mojom/mojom_parser.py b/utils/ipc/mojo/public/tools/mojom/mojom_parser.py index 12adbfb9..eb90c825 100755 --- a/utils/ipc/mojo/public/tools/mojom/mojom_parser.py +++ b/utils/ipc/mojo/public/tools/mojom/mojom_parser.py @@ -14,6 +14,8 @@ import argparse import codecs import errno import json +import logging +import multiprocessing import os import os.path import sys @@ -25,6 +27,19 @@ from mojom.parse import parser from mojom.parse import conditional_features +# Disable this for easier debugging. +# In Python 2, subprocesses just hang when exceptions are thrown :(. +_ENABLE_MULTIPROCESSING = sys.version_info[0] > 2 + +if sys.version_info < (3, 4): + _MULTIPROCESSING_USES_FORK = sys.platform.startswith('linux') +else: + # https://docs.python.org/3/library/multiprocessing.html#:~:text=bpo-33725 + if __name__ == '__main__' and sys.platform == 'darwin': + multiprocessing.set_start_method('fork') + _MULTIPROCESSING_USES_FORK = multiprocessing.get_start_method() == 'fork' + + def _ResolveRelativeImportPath(path, roots): """Attempts to resolve a relative import path against a set of possible roots. @@ -98,7 +113,7 @@ def _GetModuleFilename(mojom_filename): def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts, - dependencies, loaded_modules): + dependencies, loaded_modules, module_metadata): """Recursively ensures that a module and its dependencies are loaded. Args: @@ -111,10 +126,8 @@ def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts, by absolute file path. loaded_modules: A mapping of all modules loaded so far, including non-input modules that were pulled in as transitive dependencies of the inputs. - import_set: The working set of mojom imports processed so far in this - call stack. Used to detect circular dependencies. - import_stack: An ordered list of imports processed so far in this call - stack. Used to report circular dependencies. + module_metadata: Metadata to be attached to every module loaded by this + helper. Returns: None @@ -129,7 +142,7 @@ def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts, for dep_abspath, dep_path in dependencies[mojom_abspath]: if dep_abspath not in loaded_modules: _EnsureInputLoaded(dep_abspath, dep_path, abs_paths, asts, dependencies, - loaded_modules) + loaded_modules, module_metadata) imports = {} for imp in asts[mojom_abspath].import_list: @@ -137,6 +150,7 @@ def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts, imports[path] = loaded_modules[abs_paths[path]] loaded_modules[mojom_abspath] = translate.OrderedModule( asts[mojom_abspath], module_path, imports) + loaded_modules[mojom_abspath].metadata = dict(module_metadata) def _CollectAllowedImportsFromBuildMetadata(build_metadata_filename): @@ -157,10 +171,67 @@ def _CollectAllowedImportsFromBuildMetadata(build_metadata_filename): return allowed_imports +# multiprocessing helper. +def _ParseAstHelper(args): + mojom_abspath, enabled_features = args + with codecs.open(mojom_abspath, encoding='utf-8') as f: + ast = parser.Parse(f.read(), mojom_abspath) + conditional_features.RemoveDisabledDefinitions(ast, enabled_features) + return mojom_abspath, ast + + +# multiprocessing helper. +def _SerializeHelper(args): + mojom_abspath, mojom_path = args + module_path = os.path.join(_SerializeHelper.output_root_path, + _GetModuleFilename(mojom_path)) + module_dir = os.path.dirname(module_path) + if not os.path.exists(module_dir): + try: + # Python 2 doesn't support exist_ok on makedirs(), so we just ignore + # that failure if it happens. It's possible during build due to races + # among build steps with module outputs in the same directory. + os.makedirs(module_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + with open(module_path, 'wb') as f: + _SerializeHelper.loaded_modules[mojom_abspath].Dump(f) + + +def _Shard(target_func, args, processes=None): + args = list(args) + if processes is None: + processes = multiprocessing.cpu_count() + # Seems optimal to have each process perform at least 2 tasks. + processes = min(processes, len(args) // 2) + + if sys.platform == 'win32': + # TODO(crbug.com/1190269) - we can't use more than 56 + # cores on Windows or Python3 may hang. + processes = min(processes, 56) + + # Don't spin up processes unless there is enough work to merit doing so. + if not _ENABLE_MULTIPROCESSING or processes < 2: + for result in map(target_func, args): + yield result + return + + pool = multiprocessing.Pool(processes=processes) + try: + for result in pool.imap_unordered(target_func, args): + yield result + finally: + pool.close() + pool.join() # Needed on Windows to avoid WindowsError during terminate. + pool.terminate() + + def _ParseMojoms(mojom_files, input_root_paths, output_root_path, enabled_features, + module_metadata, allowed_imports=None): """Parses a set of mojom files and produces serialized module outputs. @@ -176,6 +247,8 @@ def _ParseMojoms(mojom_files, modules for any transitive dependencies not listed in mojom_files. enabled_features: A list of enabled feature names, controlling which AST nodes are filtered by [EnableIf] attributes. + module_metadata: A list of 2-tuples representing metadata key-value pairs to + attach to each compiled module output. Returns: None. @@ -193,72 +266,79 @@ def _ParseMojoms(mojom_files, for abs_path in mojom_files) abs_paths = dict( (path, abs_path) for abs_path, path in mojom_files_to_parse.items()) - for mojom_abspath, _ in mojom_files_to_parse.items(): - with codecs.open(mojom_abspath, encoding='utf-8') as f: - ast = parser.Parse(''.join(f.readlines()), mojom_abspath) - conditional_features.RemoveDisabledDefinitions(ast, enabled_features) - loaded_mojom_asts[mojom_abspath] = ast - invalid_imports = [] - for imp in ast.import_list: - import_abspath = _ResolveRelativeImportPath(imp.import_filename, - input_root_paths) - if allowed_imports and import_abspath not in allowed_imports: - invalid_imports.append(imp.import_filename) - - abs_paths[imp.import_filename] = import_abspath - if import_abspath in mojom_files_to_parse: - # This import is in the input list, so we're going to translate it - # into a module below; however it's also a dependency of another input - # module. We retain record of dependencies to help with input - # processing later. - input_dependencies[mojom_abspath].add((import_abspath, - imp.import_filename)) - else: - # We have an import that isn't being parsed right now. It must already - # be parsed and have a module file sitting in a corresponding output - # location. - module_path = _GetModuleFilename(imp.import_filename) - module_abspath = _ResolveRelativeImportPath(module_path, - [output_root_path]) - with open(module_abspath, 'rb') as module_file: - loaded_modules[import_abspath] = module.Module.Load(module_file) - - if invalid_imports: - raise ValueError( - '\nThe file %s imports the following files not allowed by build ' - 'dependencies:\n\n%s\n' % (mojom_abspath, - '\n'.join(invalid_imports))) + logging.info('Parsing %d .mojom into ASTs', len(mojom_files_to_parse)) + map_args = ((mojom_abspath, enabled_features) + for mojom_abspath in mojom_files_to_parse) + for mojom_abspath, ast in _Shard(_ParseAstHelper, map_args): + loaded_mojom_asts[mojom_abspath] = ast + + logging.info('Processing dependencies') + for mojom_abspath, ast in loaded_mojom_asts.items(): + invalid_imports = [] + for imp in ast.import_list: + import_abspath = _ResolveRelativeImportPath(imp.import_filename, + input_root_paths) + if allowed_imports and import_abspath not in allowed_imports: + invalid_imports.append(imp.import_filename) + + abs_paths[imp.import_filename] = import_abspath + if import_abspath in mojom_files_to_parse: + # This import is in the input list, so we're going to translate it + # into a module below; however it's also a dependency of another input + # module. We retain record of dependencies to help with input + # processing later. + input_dependencies[mojom_abspath].add( + (import_abspath, imp.import_filename)) + elif import_abspath not in loaded_modules: + # We have an import that isn't being parsed right now. It must already + # be parsed and have a module file sitting in a corresponding output + # location. + module_path = _GetModuleFilename(imp.import_filename) + module_abspath = _ResolveRelativeImportPath(module_path, + [output_root_path]) + with open(module_abspath, 'rb') as module_file: + loaded_modules[import_abspath] = module.Module.Load(module_file) + + if invalid_imports: + raise ValueError( + '\nThe file %s imports the following files not allowed by build ' + 'dependencies:\n\n%s\n' % (mojom_abspath, '\n'.join(invalid_imports))) + logging.info('Loaded %d modules from dependencies', len(loaded_modules)) # At this point all transitive imports not listed as inputs have been loaded # and we have a complete dependency tree of the unprocessed inputs. Now we can # load all the inputs, resolving dependencies among them recursively as we go. + logging.info('Ensuring inputs are loaded') num_existing_modules_loaded = len(loaded_modules) for mojom_abspath, mojom_path in mojom_files_to_parse.items(): _EnsureInputLoaded(mojom_abspath, mojom_path, abs_paths, loaded_mojom_asts, - input_dependencies, loaded_modules) + input_dependencies, loaded_modules, module_metadata) assert (num_existing_modules_loaded + len(mojom_files_to_parse) == len(loaded_modules)) # Now we have fully translated modules for every input and every transitive # dependency. We can dump the modules to disk for other tools to use. - for mojom_abspath, mojom_path in mojom_files_to_parse.items(): - module_path = os.path.join(output_root_path, _GetModuleFilename(mojom_path)) - module_dir = os.path.dirname(module_path) - if not os.path.exists(module_dir): - try: - # Python 2 doesn't support exist_ok on makedirs(), so we just ignore - # that failure if it happens. It's possible during build due to races - # among build steps with module outputs in the same directory. - os.makedirs(module_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - with open(module_path, 'wb') as f: - loaded_modules[mojom_abspath].Dump(f) + logging.info('Serializing %d modules', len(mojom_files_to_parse)) + + # Windows does not use fork() for multiprocessing, so we'd need to pass + # loaded_module via IPC rather than via globals. Doing so is slower than not + # using multiprocessing. + _SerializeHelper.loaded_modules = loaded_modules + _SerializeHelper.output_root_path = output_root_path + # Doesn't seem to help past 4. Perhaps IO bound here? + processes = 4 if _MULTIPROCESSING_USES_FORK else 0 + map_args = mojom_files_to_parse.items() + for _ in _Shard(_SerializeHelper, map_args, processes=processes): + pass def Run(command_line): + debug_logging = os.environ.get('MOJOM_PARSER_DEBUG', '0') != '0' + logging.basicConfig(level=logging.DEBUG if debug_logging else logging.WARNING, + format='%(levelname).1s %(relativeCreated)6d %(message)s') + logging.info('Started (%s)', os.path.basename(sys.argv[0])) + arg_parser = argparse.ArgumentParser( description=""" Parses one or more mojom files and produces corresponding module outputs fully @@ -333,6 +413,16 @@ already present in the provided output root.""") 'build-time dependency checking for mojom imports, where each build ' 'metadata file corresponds to a build target in the dependency graph of ' 'a typical build system.') + arg_parser.add_argument( + '--add-module-metadata', + dest='module_metadata', + default=[], + action='append', + metavar='KEY=VALUE', + help='Adds a metadata key-value pair to the output module. This can be ' + 'used by build toolchains to augment parsed mojom modules with product-' + 'specific metadata for later extraction and use by custom bindings ' + 'generators.') args, _ = arg_parser.parse_known_args(command_line) if args.mojom_file_list: @@ -353,8 +443,14 @@ already present in the provided output root.""") else: allowed_imports = None + module_metadata = list( + map(lambda kvp: tuple(kvp.split('=')), args.module_metadata)) _ParseMojoms(mojom_files, input_roots, output_root, args.enabled_features, - allowed_imports) + module_metadata, allowed_imports) + logging.info('Finished') + # Exit without running GC, which can save multiple seconds due the large + # number of object created. + os._exit(0) if __name__ == '__main__': diff --git a/utils/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py b/utils/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py index a0ee150e..65db4dc9 100644 --- a/utils/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py +++ b/utils/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py @@ -2,6 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from mojom.generate import module from mojom_parser_test_case import MojomParserTestCase @@ -20,9 +21,11 @@ class VersionCompatibilityTest(MojomParserTestCase): self.assertEqual(set(old.keys()), set(new.keys()), 'Old and new test mojoms should use the same type names.') + checker = module.BackwardCompatibilityChecker() compatibility_map = {} for name in old.keys(): - compatibility_map[name] = new[name].IsBackwardCompatible(old[name]) + compatibility_map[name] = checker.IsBackwardCompatible( + new[name], old[name]) return compatibility_map def assertBackwardCompatible(self, old_mojom, new_mojom): @@ -234,6 +237,47 @@ class VersionCompatibilityTest(MojomParserTestCase): self.assertNotBackwardCompatible('union U { string a; };', 'union U { string? a; };') + def testFieldNestedTypeChanged(self): + """Changing the definition of a nested type within a field (such as an array + element or interface endpoint type) should only break backward-compatibility + if the changes to that type are not backward-compatible.""" + self.assertBackwardCompatible( + """\ + struct S { string a; }; + struct T { array<S> ss; }; + """, """\ + struct S { + string a; + [MinVersion=1] string? b; + }; + struct T { array<S> ss; }; + """) + self.assertBackwardCompatible( + """\ + interface F { Do(); }; + struct S { pending_receiver<F> r; }; + """, """\ + interface F { + Do(); + [MinVersion=1] Say(); + }; + struct S { pending_receiver<F> r; }; + """) + + def testRecursiveTypeChange(self): + """Recursive types do not break the compatibility checker.""" + self.assertBackwardCompatible( + """\ + struct S { + string a; + array<S> others; + };""", """\ + struct S { + string a; + array<S> others; + [MinVersion=1] string? b; + };""") + def testUnionFieldBecomingNonOptional(self): """Changing a field from optional to non-optional breaks backward-compatibility.""" |