diff options
17 files changed, 852 insertions, 430 deletions
diff --git a/utils/ipc/mojo/README b/utils/ipc/mojo/README new file mode 100644 index 00000000..d5c24fc3 --- /dev/null +++ b/utils/ipc/mojo/README @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: CC0-1.0 + +Files in this directory are imported from 9c138d992bfc of Chromium. Do not +modify them manually. diff --git a/utils/ipc/mojo/public/tools/BUILD.gn b/utils/ipc/mojo/public/tools/BUILD.gn index 4c68350b..eb6391a6 100644 --- a/utils/ipc/mojo/public/tools/BUILD.gn +++ b/utils/ipc/mojo/public/tools/BUILD.gn @@ -8,11 +8,11 @@ group("mojo_python_unittests") { data = [ "run_all_python_unittests.py", - "//testing/scripts/common.py", "//testing/scripts/run_isolated_script_test.py", - "//testing/test_env.py", - "//testing/xvfb.py", ] deps = [ "//mojo/public/tools/mojom/mojom:tests" ] - data_deps = [ "//third_party/catapult/third_party/typ/" ] + data_deps = [ + "//testing:test_scripts_shared", + "//third_party/catapult/third_party/typ/", + ] } diff --git a/utils/ipc/mojo/public/tools/bindings/BUILD.gn b/utils/ipc/mojo/public/tools/bindings/BUILD.gn index 8ba6e922..3e242532 100644 --- a/utils/ipc/mojo/public/tools/bindings/BUILD.gn +++ b/utils/ipc/mojo/public/tools/bindings/BUILD.gn @@ -2,10 +2,12 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//build/config/python.gni") import("//mojo/public/tools/bindings/mojom.gni") import("//third_party/jinja2/jinja2.gni") -action("precompile_templates") { +# TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds. +python2_action("precompile_templates") { sources = mojom_generator_sources sources += [ "$mojom_generator_root/generators/cpp_templates/enum_macros.tmpl", @@ -69,11 +71,16 @@ action("precompile_templates") { "$mojom_generator_root/generators/js_templates/fuzzing.tmpl", "$mojom_generator_root/generators/js_templates/interface_definition.tmpl", "$mojom_generator_root/generators/js_templates/lite/enum_definition.tmpl", + "$mojom_generator_root/generators/js_templates/lite/enum_definition_for_module.tmpl", "$mojom_generator_root/generators/js_templates/lite/interface_definition.tmpl", + "$mojom_generator_root/generators/js_templates/lite/interface_definition_for_module.tmpl", "$mojom_generator_root/generators/js_templates/lite/module_definition.tmpl", "$mojom_generator_root/generators/js_templates/lite/mojom-lite.js.tmpl", + "$mojom_generator_root/generators/js_templates/lite/mojom.m.js.tmpl", "$mojom_generator_root/generators/js_templates/lite/struct_definition.tmpl", + "$mojom_generator_root/generators/js_templates/lite/struct_definition_for_module.tmpl", "$mojom_generator_root/generators/js_templates/lite/union_definition.tmpl", + "$mojom_generator_root/generators/js_templates/lite/union_definition_for_module.tmpl", "$mojom_generator_root/generators/js_templates/module.amd.tmpl", "$mojom_generator_root/generators/js_templates/module_definition.tmpl", "$mojom_generator_root/generators/js_templates/struct_definition.tmpl", diff --git a/utils/ipc/mojo/public/tools/bindings/README.md b/utils/ipc/mojo/public/tools/bindings/README.md index 1a3d5c58..43882450 100644 --- a/utils/ipc/mojo/public/tools/bindings/README.md +++ b/utils/ipc/mojo/public/tools/bindings/README.md @@ -113,8 +113,8 @@ for message parameters. Every Mojom file may optionally specify a single **module** to which it belongs. -This is used strictly for aggregaging all defined symbols therein within a -common Mojom namespace. The specific impact this has on generated binidngs code +This is used strictly for aggregating all defined symbols therein within a +common Mojom namespace. The specific impact this has on generated bindings code varies for each target language. For example, if the following Mojom is used to generate bindings: @@ -132,7 +132,7 @@ Generated C++ bindings will define a class interface `MoneyGenerator` in the bindings at this time are unaffected by module declarations. **NOTE:** By convention in the Chromium codebase, **all** Mojom files should -declare a module name with at least (and preferrably exactly) one top-level name +declare a module name with at least (and preferably exactly) one top-level name as well as an inner `mojom` module suffix. *e.g.*, `chrome.mojom`, `business.mojom`, *etc.* @@ -271,7 +271,7 @@ code, see ### Unions Mojom supports tagged unions using the **union** keyword. A union is a -collection of fields which may taken the value of any single one of those fields +collection of fields which may take the value of any single one of those fields at a time. Thus they provide a way to represent a variant value type while minimizing storage requirements. @@ -320,7 +320,7 @@ enum definition. By default, values are based at zero and increment by 1 sequentially. The effect of nested definitions on generated bindings varies depending on the -target language. See [documentation for individual target languages](#Generated-Code-For-Target-Languages) +target language. See [documentation for individual target languages](#Generated-Code-For-Target-Languages). ### Constants @@ -346,7 +346,7 @@ struct Employee { ``` The effect of nested definitions on generated bindings varies depending on the -target language. See [documentation for individual target languages](#Generated-Code-For-Target-Languages) +target language. See [documentation for individual target languages](#Generated-Code-For-Target-Languages). ### Interfaces @@ -379,58 +379,82 @@ Mojom definitions may have their meaning altered by **attributes**, specified with a syntax similar to Java or C# attributes. There are a handle of interesting attributes supported today. -**`[Sync]`** -: The `Sync` attribute may be specified for any interface method which expects - a response. This makes it so that callers of the method can wait - synchronously for a response. See - [Synchronous Calls](/mojo/public/cpp/bindings/README.md#Synchronous-Calls) - in the C++ bindings documentation. Note that sync methods are only actually - synchronous when called from C++. - -**`[Extensible]`** -: The `Extensible` attribute may be specified for any enum definition. This - essentially disables builtin range validation when receiving values of the - enum type in a message, allowing older bindings to tolerate unrecognized - values from newer versions of the enum. - -**`[Native]`** -: The `Native` attribute may be specified for an empty struct declaration to - provide a nominal bridge between Mojo IPC and legacy `IPC::ParamTraits` or - `IPC_STRUCT_TRAITS*` macros. - See - [Repurposing Legacy IPC Traits](/docs/mojo_ipc_conversion.md#repurposing-and-invocations) - for more details. Note support for this attribute is strictly limited to C++ - bindings generation. - -**`[MinVersion=N]`** -: The `MinVersion` attribute is used to specify the version at which a given - field, enum value, interface method, or method parameter was introduced. - See [Versioning](#Versioning) for more details. - -**`[Stable]`** -: The `Stable` attribute specifies that a given mojom type or interface - definition can be considered stable over time, meaning it is safe to use for - things like persistent storage or communication between independent - version-skewed binaries. Stable definitions may only depend on builtin mojom - types or other stable definitions, and changes to such definitions MUST - preserve backward-compatibility through appropriate use of versioning. - Backward-compatibility of changes is enforced in the Chromium tree using a - strict presubmit check. See [Versioning](#Versioning) for more details on - backward-compatibility constraints. - -**`[EnableIf=value]`** -: The `EnableIf` attribute is used to conditionally enable definitions when - the mojom is parsed. If the `mojom` target in the GN file does not include - the matching `value` in the list of `enabled_features`, the definition - will be disabled. This is useful for mojom definitions that only make - sense on one platform. Note that the `EnableIf` attribute can only be set - once per definition. +* **`[Sync]`**: + The `Sync` attribute may be specified for any interface method which expects a + response. This makes it so that callers of the method can wait synchronously + for a response. See [Synchronous + Calls](/mojo/public/cpp/bindings/README.md#Synchronous-Calls) in the C++ + bindings documentation. Note that sync methods are only actually synchronous + when called from C++. + +* **`[NoInterrupt]`**: + When a thread is waiting for a reply to a `Sync` message, it's possible to be + woken up to dispatch other unrelated incoming `Sync` messages. This measure + helps to avoid deadlocks. If a `Sync` message is also marked as `NoInterrupt` + however, this behavior is disabled: instead the calling thread will only wake + up for the precise message being waited upon. This attribute must be used with + extreme caution, because it can lead to deadlocks otherwise. + +* **`[Default]`**: + The `Default` attribute may be used to specify an enumerator value that + will be used if an `Extensible` enumeration does not deserialize to a known + value on the receiver side, i.e. the sender is using a newer version of the + enum. This allows unknown values to be mapped to a well-defined value that can + be appropriately handled. + +* **`[Extensible]`**: + The `Extensible` attribute may be specified for any enum definition. This + essentially disables builtin range validation when receiving values of the + enum type in a message, allowing older bindings to tolerate unrecognized + values from newer versions of the enum. + + Note: in the future, an `Extensible` enumeration will require that a `Default` + enumerator value also be specified. + +* **`[Native]`**: + The `Native` attribute may be specified for an empty struct declaration to + provide a nominal bridge between Mojo IPC and legacy `IPC::ParamTraits` or + `IPC_STRUCT_TRAITS*` macros. See [Repurposing Legacy IPC + Traits](/docs/mojo_ipc_conversion.md#repurposing-and-invocations) for more + details. Note support for this attribute is strictly limited to C++ bindings + generation. + +* **`[MinVersion=N]`**: + The `MinVersion` attribute is used to specify the version at which a given + field, enum value, interface method, or method parameter was introduced. + See [Versioning](#Versioning) for more details. + +* **`[Stable]`**: + The `Stable` attribute specifies that a given mojom type or interface + definition can be considered stable over time, meaning it is safe to use for + things like persistent storage or communication between independent + version-skewed binaries. Stable definitions may only depend on builtin mojom + types or other stable definitions, and changes to such definitions MUST + preserve backward-compatibility through appropriate use of versioning. + Backward-compatibility of changes is enforced in the Chromium tree using a + strict presubmit check. See [Versioning](#Versioning) for more details on + backward-compatibility constraints. + +* **`[Uuid=<UUID>]`**: + Specifies a UUID to be associated with a given interface. The UUID is intended + to remain stable across all changes to the interface definition, including + name changes. The value given for this attribute should be a standard UUID + string representation as specified by RFC 4122. New UUIDs can be generated + with common tools such as `uuidgen`. + +* **`[EnableIf=value]`**: + The `EnableIf` attribute is used to conditionally enable definitions when the + mojom is parsed. If the `mojom` target in the GN file does not include the + matching `value` in the list of `enabled_features`, the definition will be + disabled. This is useful for mojom definitions that only make sense on one + platform. Note that the `EnableIf` attribute can only be set once per + definition. ## Generated Code For Target Languages When the bindings generator successfully processes an input Mojom file, it emits corresponding code for each supported target language. For more details on how -Mojom concepts translate to a given target langauge, please refer to the +Mojom concepts translate to a given target language, please refer to the bindings API documentation for that language: * [C++ Bindings](/mojo/public/cpp/bindings/README.md) @@ -441,7 +465,7 @@ bindings API documentation for that language: Regardless of target language, all interface messages are validated during deserialization before they are dispatched to a receiving implementation of the -interface. This helps to ensure consitent validation across interfaces without +interface. This helps to ensure consistent validation across interfaces without leaving the burden to developers and security reviewers every time a new message is added. @@ -555,25 +579,37 @@ struct Employee { }; ``` -and you would like to add a birthday field. You can do: +and you would like to add birthday and nickname fields. You can add them as +optional types with a `MinVersion` like so: ``` cpp struct Employee { uint64 employee_id; string name; [MinVersion=1] Date? birthday; + [MinVersion=1] string? nickname; }; ``` +*** note +**NOTE:** Mojo object or handle types added with a `MinVersion` **MUST** be +optional (nullable). See [Primitive Types](#Primitive-Types) for details on +nullable values. +*** + By default, fields belong to version 0. New fields must be appended to the struct definition (*i.e*., existing fields must not change **ordinal value**) with the `MinVersion` attribute set to a number greater than any previous existing versions. +The value of `MinVersion` is unrelated to ordinals. The choice of a particular +version number is arbitrary. All its usage means is that a field isn't present +before the numbered version. + *** note **NOTE:** do not change existing fields in versioned structs, as this is not backwards-compatible. Instead, rename the old field to make its -deprecation clear and add a new field with the new version number. +deprecation clear and add a new field with a new `MinVersion` number. *** **Ordinal value** refers to the relative positional layout of a struct's fields @@ -602,14 +638,10 @@ struct Employee { uint64 employee_id@0; [MinVersion=1] Date? birthday@2; string name@1; + [MinVersion=1] string? nickname@3; }; ``` -*** note -**NOTE:** Newly added fields of Mojo object or handle types MUST be nullable. -See [Primitive Types](#Primitive-Types). -*** - ### Versioned Interfaces There are two dimensions on which an interface can be extended @@ -706,6 +738,39 @@ values and will need to deal with them gracefully. See [C++ Versioning Considerations](/mojo/public/cpp/bindings/README.md#Versioning-Considerations) for details. +### Renaming versioned structs +It's possible to rename versioned structs by using the `[RenamedFrom]` attribute. +RenamedFrom + +``` cpp +module asdf.mojom; + +// Old version: +[Stable] +struct OldStruct { +}; + +// New version: +[Stable, RenamedFrom="asdf.mojom.OldStruct"] +struct NewStruct { +}; +``` + +## Component targets + +If there are multiple components depending on the same mojom target within one binary, +the target will need to be defined as `mojom_component` instead of `mojom`. +Since `mojom` targets are generated `source_set` targets and `mojom_component` targets +are generated `component` targets, you would use `mojom_component` in the same cases +where you would use `component` for non-mojom files. +*** note +**NOTE**: by default, components for both blink and non-blink bindings are generated. +Use the `disable_variants` target parameter to generate only non-blink bindings. +You can also generate a `source_set` for one of the variants by defining +[export_*](https://source.chromium.org/chromium/chromium/src/+/main:mojo/public/tools/bindings/mojom.gni;drc=739b9fbce50310c1dd2b59c279cd90a9319cb6e8;l=318) +parameters for the `mojom_component` target. +*** + ## Grammar Reference Below is the (BNF-ish) context-free grammar of the Mojom language: diff --git a/utils/ipc/mojo/public/tools/bindings/gen_data_files_list.py b/utils/ipc/mojo/public/tools/bindings/gen_data_files_list.py index 79c9e50e..8b78d092 100644 --- a/utils/ipc/mojo/public/tools/bindings/gen_data_files_list.py +++ b/utils/ipc/mojo/public/tools/bindings/gen_data_files_list.py @@ -18,7 +18,6 @@ import os import re import sys -from cStringIO import StringIO from optparse import OptionParser sys.path.insert( @@ -41,12 +40,9 @@ def main(): pattern = re.compile(options.pattern) files = [f for f in os.listdir(options.directory) if pattern.match(f)] - stream = StringIO() - for f in files: - print(f, file=stream) + contents = '\n'.join(f for f in files) + '\n' + WriteFile(contents, options.output) - WriteFile(stream.getvalue(), options.output) - stream.close() if __name__ == '__main__': sys.exit(main()) diff --git a/utils/ipc/mojo/public/tools/bindings/generate_type_mappings.py b/utils/ipc/mojo/public/tools/bindings/generate_type_mappings.py index 64ca048f..a0096649 100755 --- a/utils/ipc/mojo/public/tools/bindings/generate_type_mappings.py +++ b/utils/ipc/mojo/public/tools/bindings/generate_type_mappings.py @@ -75,14 +75,6 @@ def ReadTypemap(path): return json.load(f)['c++'] -def ParseTypemapArgs(args): - typemaps = [s for s in '\n'.join(args).split('--start-typemap\n') if s] - result = {} - for typemap in typemaps: - result.update(ParseTypemap(typemap)) - return result - - def LoadCppTypemapConfig(path): configs = {} with open(path) as f: @@ -102,52 +94,6 @@ def LoadCppTypemapConfig(path): } return configs - -def ParseTypemap(typemap): - values = {'type_mappings': [], 'public_headers': [], 'traits_headers': []} - for line in typemap.split('\n'): - if not line: - continue - key, _, value = line.partition('=') - values[key].append(value.lstrip('/')) - result = {} - mapping_pattern = \ - re.compile(r"""^([^=]+) # mojom type - = - ([^[]+) # native type - (?:\[([^]]+)\])?$ # optional attribute in square brackets - """, re.X) - for typename in values['type_mappings']: - match_result = mapping_pattern.match(typename) - assert match_result, ( - "Cannot parse entry in the \"type_mappings\" section: %s" % typename) - - mojom_type = match_result.group(1) - native_type = match_result.group(2) - attributes = [] - if match_result.group(3): - attributes = match_result.group(3).split(',') - - assert mojom_type not in result, ( - "Cannot map multiple native types (%s, %s) to the same mojom type: %s" % - (result[mojom_type]['typename'], native_type, mojom_type)) - - result[mojom_type] = { - 'public_headers': values['public_headers'], - 'traits_headers': values['traits_headers'], - 'typename': native_type, - - # Attributes supported for individual mappings. - 'copyable_pass_by_value': 'copyable_pass_by_value' in attributes, - 'force_serialize': 'force_serialize' in attributes, - 'hashable': 'hashable' in attributes, - 'move_only': 'move_only' in attributes, - 'non_copyable_non_movable': 'non_copyable_non_movable' in attributes, - 'nullable_is_same_type': 'nullable_is_same_type' in attributes, - } - return result - - def main(): parser = argparse.ArgumentParser( description=__doc__, @@ -170,10 +116,10 @@ def main(): type=str, required=True, help='The path to which to write the generated JSON.') - params, typemap_params = parser.parse_known_args() - typemaps = ParseTypemapArgs(typemap_params) + params, _ = parser.parse_known_args() + typemaps = {} if params.cpp_config_path: - typemaps.update(LoadCppTypemapConfig(params.cpp_config_path)) + typemaps = LoadCppTypemapConfig(params.cpp_config_path) missing = [path for path in params.dependency if not os.path.exists(path)] if missing: raise IOError('Missing dependencies: %s' % ', '.join(missing)) diff --git a/utils/ipc/mojo/public/tools/bindings/mojom.gni b/utils/ipc/mojo/public/tools/bindings/mojom.gni index a739fa6e..fe2a1da3 100644 --- a/utils/ipc/mojo/public/tools/bindings/mojom.gni +++ b/utils/ipc/mojo/public/tools/bindings/mojom.gni @@ -2,7 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import("//build/config/jumbo.gni") +import("//build/config/python.gni") import("//third_party/closure_compiler/closure_args.gni") import("//third_party/closure_compiler/compile_js.gni") import("//third_party/protobuf/proto_library.gni") @@ -64,10 +64,13 @@ declare_args() { # ARC) we have to explicitly opt out there even when NaCl is enabled (and # consequently also when building for NaCl toolchains.) For this reason we # check |target_os| explicitly, as it's consistent across all toolchains. +# +# TODO(crbug.com/1052397): Remove !chromeos_is_browser_only once +# lacros-chrome switches to target_os="chromeos" enable_scrambled_message_ids = enable_mojom_message_id_scrambling && - (is_mac || is_win || (is_linux && !is_chromeos && !is_chromecast && - !chromeos_is_browser_only) || + (is_mac || is_win || + (is_linux && !is_chromeos_ash && !is_chromecast && !is_chromeos_lacros) || ((enable_nacl || is_nacl || is_nacl_nonsfi) && (target_os != "chromeos" && !chromeos_is_browser_only))) @@ -78,7 +81,6 @@ mojom_parser_sources = [ "$_mojom_library_root/__init__.py", "$_mojom_library_root/error.py", "$_mojom_library_root/generate/__init__.py", - "$_mojom_library_root/generate/constant_resolver.py", "$_mojom_library_root/generate/generator.py", "$_mojom_library_root/generate/module.py", "$_mojom_library_root/generate/pack.py", @@ -94,6 +96,7 @@ mojom_generator_root = "$_mojom_tools_root/bindings" mojom_generator_script = "$mojom_generator_root/mojom_bindings_generator.py" mojom_generator_sources = mojom_parser_sources + [ + "$mojom_generator_root/generators/cpp_util.py", "$mojom_generator_root/generators/mojom_cpp_generator.py", "$mojom_generator_root/generators/mojom_java_generator.py", "$mojom_generator_root/generators/mojom_mojolpm_generator.py", @@ -107,17 +110,6 @@ if (enable_scrambled_message_ids) { # The path to a file whose contents can be used as the basis for a message # ID scrambling salt. mojom_message_id_salt_path = "//chrome/VERSION" - - # The path to a file whose contents will be concatenated to the contents of - # the file at |mojom_message_id_salt_path| to form a complete salt for - # message ID scrambling. May be the empty string, in which case the contents - # of the above file alone are used as the complete salt. - if (is_chrome_branded) { - mojom_message_id_salt_suffix_path = - "//mojo/internal/chrome-message-id-salt-suffix" - } else { - mojom_message_id_salt_suffix_path = "" - } } assert(mojom_message_id_salt_path != "") @@ -126,56 +118,11 @@ if (enable_scrambled_message_ids) { rebase_path(mojom_message_id_salt_path, root_build_dir), ] message_scrambling_inputs = [ mojom_message_id_salt_path ] - - if (mojom_message_id_salt_suffix_path != "") { - message_scrambling_args += [ - "--scrambled_message_id_salt_path", - rebase_path(mojom_message_id_salt_suffix_path, root_build_dir), - ] - message_scrambling_inputs += [ mojom_message_id_salt_suffix_path ] - } } else { message_scrambling_args = [] message_scrambling_inputs = [] } -if (enable_mojom_typemapping) { - _bindings_configuration_files = - [ "//mojo/public/tools/bindings/chromium_bindings_configuration.gni" ] - _bindings_configurations = [] - foreach(config_file, _bindings_configuration_files) { - _bindings_configurations += [ read_file(config_file, "scope") ] - } - foreach(configuration, _bindings_configurations) { - # Check that the mojom field of each typemap refers to a mojom that exists. - foreach(typemap, configuration.typemaps) { - _typemap_config = { - } - _typemap_config = typemap.config - read_file(_typemap_config.mojom, "") - } - } -} else { - _bindings_configuration_files = [] - _bindings_configurations = [ - { - typemaps = [] - component_macro_suffix = "" - }, - ] -} - -if (!is_ios) { - _bindings_configurations += [ - { - variant = "blink" - component_macro_suffix = "_BLINK" - for_blink = true - typemaps = [] - }, - ] -} - # Generates targets for building C++, JavaScript and Java bindings from mojom # files. The output files will go under the generated file directory tree with # the same path as each input file. @@ -218,6 +165,15 @@ if (!is_ios) { # import_dirs (optional) # List of import directories that will get added when processing sources. # +# input_root_override (optional) +# Root path for the .mojom files used to generate the namespaces for +# interfaces. Useful with targets outside //, e.g. in parent directories +# above "//". The default input root is // +# Example: Vivaldi's source root is "//vivaldi/", +# and "//vivaldi/chromium/" is "//" +# In such cases, not using this argument lead to the output files being +# located in different directories than expected. +# # testonly (optional) # # visibility (optional) @@ -245,6 +201,43 @@ if (!is_ios) { # blink_cpp_typemaps (optional) # Same as above, but for the Blink variant of generated C++ bindings. # +# cpp_proxy_target (optional) +# The name of a target which all C++ dependencies will link against +# instead of linking directly against this mojom target's generated C++ +# sources. Normally when declaring invoking the mojom("foo") target, GN +# emits a source_set or component target named "foo" which encompasses the +# default variant of generated C++ bindings. This changes that to instead +# emit a group("foo") which merely forwards public_deps to the named +# `cpp_proxy_target`. That target must in turn depend on +# "foo_cpp_sources". +# +# This is useful primarily in conjunction with export_define et al to +# embed generated C++ bindings within an existing component target. +# +# blink_cpp_proxy_target (optional) +# Same concept as `cpp_proxy_target` above, but affects the generated +# "foo_blink" Blink-variant C++ bindings. +# +# cpp_configs (optional) +# A list of extra configs to apply to the default variant of generated C++ +# bindings. +# +# blink_cpp_configs (optional) +# A list of extra configs to apply to the Blink variant of generated C++ +# bindings. +# +# mojom_source_deps (optional) +# A list of mojoms this target depends upon. This is equivalent to +# public_deps except that the C++ bindings depend on each of the named +# "foo" targets' "foo_cpp_sources" rather than on foo's +# `cpp_proxy_target`. It only makes sense to use this for dependencies +# that set `cpp_proxy_target`, and only when the dependent mojom() would +# otherwise have circular dependencies with that proxy target. +# +# mojom_blink_source_deps (optional) +# Same as above but depends on "foo_blink_cpp_sources" and is used for +# dependencies that specify a `blink_cpp_proxy_target`. +# # generate_java (optional) # If set to true, Java bindings are generated for Android builds. If # |cpp_only| is set to true, it overrides this to prevent generation of @@ -327,6 +320,21 @@ if (!is_ios) { # List of extra C++ templates that are used to generate additional source # and/or header files. The templates should end with extension ".tmpl". # +# webui_module_path (optional) +# The path or URL at which modules generated by this target will be +# accessible to WebUI pages. This may either be an absolute path or +# a full URL path starting with "chrome://resources/mojo". +# +# If an absolute path, a WebUI page may only import these modules if +# they are manually packaged and mapped independently by that page's +# WebUIDataSource. The mapped path must match the path given here. +# +# If this is is instead a URL string starting with +# "chrome://resources/mojo", the generated resources must be added to +# content_resources.grd and registered with +# content::SharedResourcesDataSource with a corresponding path, at which +# point they will be made available to all WebUI pages at the given URL. +# # The following parameters are used to support the component build. They are # needed so that bindings which are linked with a component can use the same # export settings for classes. The first three are for the chromium variant, and @@ -410,8 +418,8 @@ if (!is_ios) { # traits for the type must define IsNull and SetToNull methods. # # When false, nullable fields are represented by wrapping the C++ -# type with base::Optional, and null values are simply -# base::nullopt. +# type with absl::optional, and null values are simply +# absl::nullopt. # # hashable (optional) # A boolean value (default false) indicating whether the C++ type is @@ -463,14 +471,15 @@ template("mojom") { if (defined(invoker.export_class_attribute) || defined(invoker.export_define) || defined(invoker.export_header)) { assert(defined(invoker.export_class_attribute)) - assert(defined(invoker.export_define)) + assert(defined(invoker.export_define) || defined(invoker.cpp_configs)) assert(defined(invoker.export_header)) } if (defined(invoker.export_class_attribute_blink) || defined(invoker.export_define_blink) || defined(invoker.export_header_blink)) { assert(defined(invoker.export_class_attribute_blink)) - assert(defined(invoker.export_define_blink)) + assert(defined(invoker.export_define_blink) || + defined(invoker.blink_cpp_configs)) assert(defined(invoker.export_header_blink)) # Not all platforms use the Blink variant, so make sure GN doesn't complain @@ -506,12 +515,22 @@ template("mojom") { !invoker.disallow_interfaces all_deps = [] + mojom_cpp_deps = [] if (defined(invoker.deps)) { all_deps += invoker.deps + mojom_cpp_deps += invoker.deps } if (defined(invoker.public_deps)) { all_deps += invoker.public_deps + mojom_cpp_deps += invoker.public_deps + } + if (defined(invoker.mojom_source_deps)) { + all_deps += invoker.mojom_source_deps } + if (defined(invoker.mojom_blink_source_deps)) { + all_deps += invoker.mojom_blink_source_deps + } + not_needed([ "mojom_deps" ]) if (defined(invoker.component_macro_prefix)) { assert(defined(invoker.component_output_prefix)) @@ -536,12 +555,6 @@ template("mojom") { sources_list = invoker.sources } - # Reset sources_assignment_filter for the BUILD.gn file to prevent - # regression during the migration of Chromium away from the feature. - # See docs/no_sources_assignment_filter.md for more information. - # TODO(crbug.com/1018739): remove this when migration is done. - set_sources_assignment_filter([]) - # Listed sources may be relative to the current target dir, or they may be # absolute paths, including paths to generated mojom files. While those are # fine as-is for input references, deriving output paths can be more subtle. @@ -571,7 +584,11 @@ template("mojom") { # for the naming of any generated output files derived from their # corresponding input mojoms. These paths are always considered to be relative # to root_gen_dir. - source_abspaths = rebase_path(sources_list, "//") + if (defined(invoker.input_root_override)) { + source_abspaths = rebase_path(sources_list, invoker.input_root_override) + } else { + source_abspaths = rebase_path(sources_list, "//") + } output_file_base_paths = [] foreach(path, source_abspaths) { output_file_base_paths += @@ -643,21 +660,31 @@ template("mojom") { } if (is_android) { enabled_features += [ "is_android" ] - } else if (is_chromeos) { - enabled_features += [ "is_chromeos" ] + } else if (is_chromeos_ash) { + enabled_features += [ + "is_chromeos", + "is_chromeos_ash", + ] } else if (is_fuchsia) { enabled_features += [ "is_fuchsia" ] } else if (is_ios) { enabled_features += [ "is_ios" ] - } else if (is_linux) { + } else if (is_linux || is_chromeos_lacros) { enabled_features += [ "is_linux" ] + if (is_chromeos_lacros) { + enabled_features += [ + "is_chromeos", + "is_chromeos_lacros", + ] + } } else if (is_mac) { enabled_features += [ "is_mac" ] } else if (is_win) { enabled_features += [ "is_win" ] } - action(parser_target_name) { + # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds. + python2_action(parser_target_name) { script = mojom_parser_script inputs = mojom_parser_sources + [ build_metadata_filename ] sources = sources_list @@ -679,7 +706,7 @@ template("mojom") { # Resolve relative input mojom paths against both the root src dir and # the root gen dir. "--input-root", - rebase_path("//"), + rebase_path("//."), "--input-root", rebase_path(root_gen_dir), @@ -692,12 +719,26 @@ template("mojom") { rebase_path(build_metadata_filename), ] + if (defined(invoker.input_root_override)) { + args += [ + "--input-root", + rebase_path(invoker.input_root_override), + ] + } + foreach(enabled_feature, enabled_features) { args += [ "--enable-feature", enabled_feature, ] } + + if (defined(invoker.webui_module_path)) { + args += [ + "--add-module-metadata", + "webui_module_path=${invoker.webui_module_path}", + ] + } } } @@ -705,18 +746,29 @@ template("mojom") { # Generate code that is shared by different variants. if (sources_list != []) { + base_dir = "//" + if (defined(invoker.input_root_override)) { + base_dir = invoker.input_root_override + } + common_generator_args = [ "--use_bundled_pylibs", "-o", rebase_path(root_gen_dir, root_build_dir), "generate", "-d", - rebase_path("//", root_build_dir), + rebase_path(base_dir, root_build_dir), "-I", rebase_path("//", root_build_dir), "--bytecode_path", rebase_path("$root_gen_dir/mojo/public/tools/bindings", root_build_dir), ] + if (defined(invoker.input_root_override)) { + common_generator_args += [ + "-I", + rebase_path(invoker.input_root_override, root_build_dir), + ] + } if (defined(invoker.disallow_native_types) && invoker.disallow_native_types) { @@ -767,7 +819,8 @@ template("mojom") { } } - action(generator_cpp_message_ids_target_name) { + # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds. + python2_action(generator_cpp_message_ids_target_name) { script = mojom_generator_script inputs = mojom_generator_sources + jinja2_sources sources = sources_list @@ -806,7 +859,9 @@ template("mojom") { } generator_shared_target_name = "${target_name}_shared__generator" - action(generator_shared_target_name) { + + # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds. + python2_action(generator_shared_target_name) { visibility = [ ":*" ] script = mojom_generator_script inputs = mojom_generator_sources + jinja2_sources @@ -864,7 +919,7 @@ template("mojom") { } shared_cpp_sources_target_name = "${target_name}_shared_cpp_sources" - jumbo_source_set(shared_cpp_sources_target_name) { + source_set(shared_cpp_sources_target_name) { if (defined(invoker.testonly)) { testonly = invoker.testonly } @@ -925,7 +980,9 @@ template("mojom") { generator_mojolpm_proto_target_name = "${target_name}_mojolpm_proto_generator" - action(generator_mojolpm_proto_target_name) { + + # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds. + python2_action(generator_mojolpm_proto_target_name) { script = mojom_generator_script inputs = mojom_generator_sources + jinja2_sources sources = invoker.sources @@ -960,11 +1017,10 @@ template("mojom") { sources = process_file_template( invoker.sources, [ "{{source_gen_dir}}/{{source_file_part}}.mojolpm.proto" ]) - import_dirs = [ "${root_gen_dir}" ] + import_dirs = [ "//" ] proto_in_dir = "${root_gen_dir}" proto_out_dir = "." - proto_deps = [ "//mojo/public/tools/fuzzers:mojolpm_proto_copy" ] - proto_deps += [ ":$generator_mojolpm_proto_target_name" ] + proto_deps = [ ":$generator_mojolpm_proto_target_name" ] link_deps = [ "//mojo/public/tools/fuzzers:mojolpm_proto" ] foreach(d, all_deps) { @@ -995,12 +1051,22 @@ template("mojom") { } # Generate code for variants. - if (!defined(invoker.disable_variants) || !invoker.disable_variants) { - enabled_configurations = _bindings_configurations + default_variant = { + component_macro_suffix = "" + } + if ((!defined(invoker.disable_variants) || !invoker.disable_variants) && + !is_ios) { + blink_variant = { + variant = "blink" + component_macro_suffix = "_BLINK" + for_blink = true + } + enabled_configurations = [ + default_variant, + blink_variant, + ] } else { - first_config = _bindings_configurations[0] - assert(!defined(first_config.variant)) - enabled_configurations = [ first_config ] + enabled_configurations = [ default_variant ] } foreach(bindings_configuration, enabled_configurations) { cpp_only = false @@ -1018,6 +1084,11 @@ template("mojom") { export_defines = [] export_defines_overridden = false force_source_set = false + proxy_target = "" + extra_configs = [] + output_visibility = [] + output_visibility = [ "*" ] + cpp_source_deps = [] if (defined(bindings_configuration.for_blink) && bindings_configuration.for_blink) { if (defined(invoker.blink_cpp_typemaps)) { @@ -1028,18 +1099,47 @@ template("mojom") { export_defines = [ invoker.export_define_blink ] force_source_set = true } + if (defined(invoker.blink_cpp_configs)) { + extra_configs += invoker.blink_cpp_configs + } + if (defined(invoker.blink_cpp_proxy_target)) { + proxy_target = invoker.blink_cpp_proxy_target + } + if (defined(invoker.visibility_blink)) { + output_visibility = [] + output_visibility = invoker.visibility_blink + } + if (defined(invoker.mojom_blink_source_deps)) { + cpp_source_deps = invoker.mojom_blink_source_deps + } } else { if (defined(invoker.cpp_typemaps)) { cpp_typemap_configs = invoker.cpp_typemaps } - if (defined(invoker.export_define)) { export_defines_overridden = true export_defines = [ invoker.export_define ] force_source_set = true } + if (defined(invoker.cpp_configs)) { + extra_configs += invoker.cpp_configs + } + if (defined(invoker.cpp_proxy_target)) { + proxy_target = invoker.cpp_proxy_target + } + if (defined(invoker.visibility)) { + output_visibility = [] + output_visibility = invoker.visibility + } + if (defined(invoker.mojom_source_deps)) { + cpp_source_deps = invoker.mojom_source_deps + } } not_needed([ "cpp_typemap_configs" ]) + if (proxy_target != "") { + group("${target_name}${variant_suffix}__has_cpp_proxy") { + } + } if (!export_defines_overridden && defined(invoker.component_macro_prefix)) { output_name_override = @@ -1089,7 +1189,6 @@ template("mojom") { type_mappings_target_name = "${target_name}${variant_suffix}__type_mappings" type_mappings_path = "$target_gen_dir/${target_name}${variant_suffix}__type_mappings" - active_typemaps = [] if (sources_list != []) { generator_cpp_output_suffixes = [] variant_dash_suffix = "" @@ -1104,20 +1203,11 @@ template("mojom") { "${variant_dash_suffix}.cc", "${variant_dash_suffix}.h", ] - foreach(source, sources_list) { - # TODO(sammc): Use a map instead of a linear scan when GN supports maps. - foreach(typemap, bindings_configuration.typemaps) { - _typemap_config = { - } - _typemap_config = typemap.config - if (get_path_info(source, "abspath") == _typemap_config.mojom) { - active_typemaps += [ typemap ] - } - } - } generator_target_name = "${target_name}${variant_suffix}__generator" - action(generator_target_name) { + + # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds. + python2_action(generator_target_name) { visibility = [ ":*" ] script = mojom_generator_script inputs = mojom_generator_sources + jinja2_sources @@ -1249,6 +1339,7 @@ template("mojom") { # mojolpm only uses the no-variant type. ":$mojolpm_generator_target_name", ":$mojolpm_proto_target_name", + "//base", "//mojo/public/tools/fuzzers:mojolpm", ] @@ -1260,18 +1351,6 @@ template("mojom") { public_deps += [ "${full_name}_mojolpm" ] } - foreach(typemap, active_typemaps) { - _typemap_config = { - } - _typemap_config = typemap.config - - if (defined(_typemap_config.deps)) { - deps += _typemap_config.deps - } - if (defined(_typemap_config.public_deps)) { - public_deps += _typemap_config.public_deps - } - } foreach(config, cpp_typemap_configs) { if (defined(config.traits_deps)) { deps += config.traits_deps @@ -1309,7 +1388,9 @@ template("mojom") { } write_file(_typemap_config_filename, _rebased_typemap_configs, "json") _mojom_target_name = target_name - action(_typemap_validator_target_name) { + + # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds. + python2_action(_typemap_validator_target_name) { script = "$mojom_generator_root/validate_typemap_config.py" inputs = [ _typemap_config_filename ] outputs = [ _typemap_stamp_filename ] @@ -1320,9 +1401,10 @@ template("mojom") { ] } - action(type_mappings_target_name) { - inputs = _bindings_configuration_files + mojom_generator_sources + - jinja2_sources + [ _typemap_stamp_filename ] + # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds. + python2_action(type_mappings_target_name) { + inputs = + mojom_generator_sources + jinja2_sources + [ _typemap_stamp_filename ] outputs = [ type_mappings_path ] script = "$mojom_generator_root/generate_type_mappings.py" deps = [ ":$_typemap_validator_target_name" ] @@ -1349,46 +1431,12 @@ template("mojom") { ] } - if (sources_list != []) { - # TODO(sammc): Pass the typemap description in a file to avoid command - # line length limitations. - typemap_description = [] - foreach(typemap, active_typemaps) { - _typemap_config = { - } - _typemap_config = typemap.config - - typemap_description += [ "--start-typemap" ] - if (defined(_typemap_config.public_headers)) { - foreach(value, _typemap_config.public_headers) { - typemap_description += [ "public_headers=$value" ] - } - } - if (defined(_typemap_config.traits_headers)) { - foreach(value, _typemap_config.traits_headers) { - typemap_description += [ "traits_headers=$value" ] - } - } - foreach(value, _typemap_config.type_mappings) { - typemap_description += [ "type_mappings=$value" ] - } - - # The typemap configuration files are not actually used as inputs here - # but this establishes a necessary build dependency to ensure that - # typemap changes force a rebuild of affected targets. - if (defined(typemap.filename)) { - inputs += [ typemap.filename ] - } - } - args += typemap_description - - # Newer GN-based typemaps are aggregated into a single config. - inputs += [ _typemap_config_filename ] - args += [ - "--cpp-typemap-config", - rebase_path(_typemap_config_filename, root_build_dir), - ] - } + # Newer GN-based typemaps are aggregated into a single config. + inputs += [ _typemap_config_filename ] + args += [ + "--cpp-typemap-config", + rebase_path(_typemap_config_filename, root_build_dir), + ] } group("${target_name}${variant_suffix}_headers") { @@ -1404,32 +1452,45 @@ template("mojom") { full_name = get_label_info("$d", "label_no_toolchain") public_deps += [ "${full_name}${variant_suffix}_headers" ] } + if (defined(bindings_configuration.for_blink) && + bindings_configuration.for_blink) { + public_deps += [ "//mojo/public/cpp/bindings:wtf_support" ] + } } + js_data_deps_target_name = target_name + "_js_data_deps" + not_needed([ "js_data_deps_target_name" ]) + if (!force_source_set && defined(invoker.component_macro_prefix)) { - output_target_type = "component" + sources_target_type = "component" } else { - output_target_type = "source_set" + sources_target_type = "source_set" } - js_data_deps_target_name = target_name + "_js_data_deps" - not_needed([ "js_data_deps_target_name" ]) + output_target_name = "${target_name}${variant_suffix}" + if (proxy_target != "") { + group(output_target_name) { + public_deps = [ proxy_target ] + visibility = output_visibility + if (defined(invoker.testonly)) { + testonly = invoker.testonly + } + } + sources_target_name = "${output_target_name}_cpp_sources" + } else { + sources_target_name = output_target_name + } - target("jumbo_" + output_target_type, "${target_name}${variant_suffix}") { + target(sources_target_type, sources_target_name) { if (defined(output_name_override)) { output_name = output_name_override } - if (defined(bindings_configuration.for_blink) && - bindings_configuration.for_blink && - defined(invoker.visibility_blink)) { - visibility = invoker.visibility_blink - } else if (defined(invoker.visibility)) { - visibility = invoker.visibility - } + visibility = output_visibility + [ ":$output_target_name" ] if (defined(invoker.testonly)) { testonly = invoker.testonly } defines = export_defines + configs += extra_configs if (output_file_base_paths != []) { sources = [] foreach(base_path, output_file_base_paths) { @@ -1456,13 +1517,20 @@ template("mojom") { if (sources_list != []) { public_deps += [ ":$generator_target_name" ] } - foreach(d, all_deps) { + foreach(d, mojom_cpp_deps) { # Resolve the name, so that a target //mojo/something becomes # //mojo/something:something and we can append variant_suffix to # get the cpp dependency name. - full_name = get_label_info("$d", "label_no_toolchain") + full_name = get_label_info(d, "label_no_toolchain") public_deps += [ "${full_name}${variant_suffix}" ] } + foreach(d, cpp_source_deps) { + full_name = get_label_info(d, "label_no_toolchain") + public_deps += [ + "${full_name}${variant_suffix}__has_cpp_proxy", + "${full_name}${variant_suffix}_cpp_sources", + ] + } if (defined(bindings_configuration.for_blink) && bindings_configuration.for_blink) { if (defined(invoker.overridden_deps_blink)) { @@ -1493,20 +1561,6 @@ template("mojom") { public_deps += invoker.component_deps } } - foreach(typemap, active_typemaps) { - _typemap_config = { - } - _typemap_config = typemap.config - if (defined(_typemap_config.sources)) { - sources += _typemap_config.sources - } - if (defined(_typemap_config.public_deps)) { - public_deps += _typemap_config.public_deps - } - if (defined(_typemap_config.deps)) { - deps += _typemap_config.deps - } - } foreach(config, cpp_typemap_configs) { if (defined(config.traits_sources)) { sources += config.traits_sources @@ -1518,18 +1572,10 @@ template("mojom") { public_deps += config.traits_public_deps } } - if (defined(invoker.export_header)) { - sources += [ "//" + invoker.export_header ] - } if (defined(bindings_configuration.for_blink) && bindings_configuration.for_blink) { public_deps += [ "//mojo/public/cpp/bindings:wtf_support" ] } - - if (generate_fuzzing) { - # Generate JS bindings by default if IPC fuzzer is enabled. - public_deps += [ ":$js_data_deps_target_name" ] - } } if (generate_java && is_android) { @@ -1537,7 +1583,8 @@ template("mojom") { java_generator_target_name = target_name + "_java__generator" if (sources_list != []) { - action(java_generator_target_name) { + # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds. + python2_action(java_generator_target_name) { script = mojom_generator_script inputs = mojom_generator_sources + jinja2_sources sources = sources_list @@ -1576,7 +1623,9 @@ template("mojom") { } java_srcjar_target_name = target_name + "_java_sources" - action(java_srcjar_target_name) { + + # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds. + python2_action(java_srcjar_target_name) { script = "//build/android/gyp/zip.py" inputs = [] if (output_file_base_paths != []) { @@ -1605,6 +1654,7 @@ template("mojom") { "//base:base_java", "//mojo/public/java:bindings_java", "//mojo/public/java:system_java", + "//third_party/androidx:androidx_annotation_annotation_java", ] # Disable warnings/checks on these generated files. @@ -1635,7 +1685,9 @@ template("mojom") { !use_typescript_for_target) { if (sources_list != []) { generator_js_target_name = "${target_name}_js__generator" - action(generator_js_target_name) { + + # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds. + python2_action(generator_js_target_name) { script = mojom_generator_script inputs = mojom_generator_sources + jinja2_sources sources = sources_list @@ -1656,10 +1708,15 @@ template("mojom") { outputs += [ "$root_gen_dir/$base_path.js", "$root_gen_dir/$base_path.externs.js", + "$root_gen_dir/$base_path.m.js", "$root_gen_dir/$base_path-lite.js", "$root_gen_dir/$base_path.html", "$root_gen_dir/$base_path-lite-for-compile.js", ] + + if (defined(invoker.webui_module_path)) { + outputs += [ "$root_gen_dir/mojom-webui/$base_path-webui.js" ] + } } response_file_contents = filelist @@ -1708,13 +1765,19 @@ template("mojom") { foreach(base_path, output_file_base_paths) { data += [ "$root_gen_dir/${base_path}.js", + "$root_gen_dir/${base_path}.m.js", "$root_gen_dir/${base_path}-lite.js", ] } deps += [ ":$generator_js_target_name" ] } - data_deps = [] + if (defined(invoker.disallow_native_types) && + invoker.disallow_native_types) { + data_deps = [] + } else { + data_deps = [ "//mojo/public/js:bindings_module" ] + } foreach(d, all_deps) { full_name = get_label_info(d, "label_no_toolchain") data_deps += [ "${full_name}_js_data_deps" ] @@ -1770,6 +1833,64 @@ template("mojom") { group(js_library_for_compile_target_name) { } } + + js_modules_target_name = "${target_name}_js_modules" + if (sources_list != []) { + js_library(js_modules_target_name) { + extra_public_deps = [ ":$generator_js_target_name" ] + sources = [] + foreach(base_path, output_file_base_paths) { + sources += [ "$root_gen_dir/${base_path}.m.js" ] + } + externs_list = [ + "${externs_path}/mojo_core.js", + "${externs_path}/pending.js", + ] + if (defined(invoker.disallow_native_types) && + invoker.disallow_native_types) { + deps = [] + } else { + deps = [ "//mojo/public/js:bindings_uncompiled" ] + } + foreach(d, all_deps) { + full_name = get_label_info(d, "label_no_toolchain") + deps += [ "${full_name}_js_modules" ] + } + } + } else { + group(js_modules_target_name) { + } + } + + if (defined(invoker.webui_module_path)) { + webui_js_target_name = "${target_name}_webui_js" + if (sources_list != []) { + js_library(webui_js_target_name) { + extra_public_deps = [ ":$generator_js_target_name" ] + sources = [] + foreach(base_path, output_file_base_paths) { + sources += [ "$root_gen_dir/mojom-webui/${base_path}-webui.js" ] + } + externs_list = [ + "${externs_path}/mojo_core.js", + "${externs_path}/pending.js", + ] + if (defined(invoker.disallow_native_types) && + invoker.disallow_native_types) { + deps = [] + } else { + deps = [ "//mojo/public/js:bindings_uncompiled" ] + } + foreach(d, all_deps) { + full_name = get_label_info(d, "label_no_toolchain") + deps += [ "${full_name}_webui_js" ] + } + } + } else { + group(webui_js_target_name) { + } + } + } } if ((generate_fuzzing || !defined(invoker.cpp_only) || !invoker.cpp_only) && use_typescript_for_target) { @@ -1806,7 +1927,9 @@ template("mojom") { # Generate Typescript bindings. generator_ts_target_name = "${target_name}_${dependency_type.name}__ts__generator" - action(generator_ts_target_name) { + + # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds. + python2_action(generator_ts_target_name) { script = mojom_generator_script inputs = mojom_generator_sources + jinja2_sources sources = sources_list @@ -1873,7 +1996,8 @@ template("mojom") { "${target_name}_${dependency_type.name}__js__generator" generator_js_target_names += [ generator_js_target_name ] - action(generator_js_target_name) { + # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds. + python2_action(generator_js_target_name) { script = "$mojom_generator_root/compile_typescript.py" sources = ts_outputs outputs = js_outputs diff --git a/utils/ipc/mojo/public/tools/mojom/README.md b/utils/ipc/mojo/public/tools/mojom/README.md index 6a4ff78a..e5d17ab0 100644 --- a/utils/ipc/mojo/public/tools/mojom/README.md +++ b/utils/ipc/mojo/public/tools/mojom/README.md @@ -3,7 +3,7 @@ The Mojom format is an interface definition language (IDL) for describing interprocess communication (IPC) messages and data types for use with the low-level cross-platform -[Mojo IPC library](https://chromium.googlesource.com/chromium/src/+/master/mojo/public/c/system/README.md). +[Mojo IPC library](https://chromium.googlesource.com/chromium/src/+/main/mojo/public/c/system/README.md). This directory consists of a `mojom` Python module, its tests, and supporting command-line tools. The Python module implements the parser used by the diff --git a/utils/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py b/utils/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py index 7e746112..08bd672f 100755 --- a/utils/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py +++ b/utils/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py @@ -131,7 +131,8 @@ def _ValidateDelta(root, delta): 'renamed, please add a [RenamedFrom] attribute to the new type. This ' 'can be deleted by a subsequent change.' % qualified_name) - if not new_types[new_name].IsBackwardCompatible(kind): + checker = module.BackwardCompatibilityChecker() + if not checker.IsBackwardCompatible(new_types[new_name], kind): raise Exception('Stable type %s appears to have changed in a way which ' 'breaks backward-compatibility. Please fix!\n\nIf you ' 'believe this assessment to be incorrect, please file a ' diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/BUILD.gn b/utils/ipc/mojo/public/tools/mojom/mojom/BUILD.gn index 7416ef19..51facc0c 100644 --- a/utils/ipc/mojo/public/tools/mojom/mojom/BUILD.gn +++ b/utils/ipc/mojo/public/tools/mojom/mojom/BUILD.gn @@ -8,7 +8,6 @@ group("mojom") { "error.py", "fileutil.py", "generate/__init__.py", - "generate/constant_resolver.py", "generate/generator.py", "generate/module.py", "generate/pack.py", diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/generate/generator.py b/utils/ipc/mojo/public/tools/mojom/mojom/generate/generator.py index de62260a..4a1c73fc 100644 --- a/utils/ipc/mojo/public/tools/mojom/mojom/generate/generator.py +++ b/utils/ipc/mojo/public/tools/mojom/mojom/generate/generator.py @@ -136,9 +136,14 @@ class Stylizer(object): def WriteFile(contents, full_path): # If |contents| is same with the file content, we skip updating. + if not isinstance(contents, bytes): + data = contents.encode('utf8') + else: + data = contents + if os.path.isfile(full_path): with open(full_path, 'rb') as destination_file: - if destination_file.read() == contents: + if destination_file.read() == data: return # Make sure the containing directory exists. @@ -146,11 +151,8 @@ def WriteFile(contents, full_path): fileutil.EnsureDirectoryExists(full_dir) # Dump the data to disk. - with open(full_path, "wb") as f: - if not isinstance(contents, bytes): - f.write(contents.encode('utf-8')) - else: - f.write(contents) + with open(full_path, 'wb') as f: + f.write(data) def AddComputedData(module): diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/generate/module.py b/utils/ipc/mojo/public/tools/mojom/mojom/generate/module.py index 8547ff64..9bdb28e0 100644 --- a/utils/ipc/mojo/public/tools/mojom/mojom/generate/module.py +++ b/utils/ipc/mojo/public/tools/mojom/mojom/generate/module.py @@ -12,7 +12,33 @@ # method = interface.AddMethod('Tat', 0) # method.AddParameter('baz', 0, mojom.INT32) -import pickle +import sys +if sys.version_info.major == 2: + import cPickle as pickle +else: + import pickle +from uuid import UUID + + +class BackwardCompatibilityChecker(object): + """Used for memoization while recursively checking two type definitions for + backward-compatibility.""" + + def __init__(self): + self._cache = {} + + def IsBackwardCompatible(self, new_kind, old_kind): + key = (new_kind, old_kind) + result = self._cache.get(key) + if result is None: + # Assume they're compatible at first to effectively ignore recursive + # checks between these types, e.g. if both kinds are a struct or union + # that references itself in a field. + self._cache[key] = True + result = new_kind.IsBackwardCompatible(old_kind, self) + self._cache[key] = result + return result + # We use our own version of __repr__ when displaying the AST, as the # AST currently doesn't capture which nodes are reference (e.g. to @@ -114,6 +140,10 @@ class Kind(object): # during a subsequent run of the parser. return hash((self.spec, self.parent_kind)) + # pylint: disable=unused-argument + def IsBackwardCompatible(self, rhs, checker): + return self == rhs + class ReferenceKind(Kind): """ReferenceKind represents pointer and handle types. @@ -195,6 +225,10 @@ class ReferenceKind(Kind): def __hash__(self): return hash((super(ReferenceKind, self).__hash__(), self.is_nullable)) + def IsBackwardCompatible(self, rhs, checker): + return (super(ReferenceKind, self).IsBackwardCompatible(rhs, checker) + and self.is_nullable == rhs.is_nullable) + # Initialize the set of primitive types. These can be accessed by clients. BOOL = Kind('b') @@ -253,9 +287,13 @@ PRIMITIVES = ( ) ATTRIBUTE_MIN_VERSION = 'MinVersion' +ATTRIBUTE_DEFAULT = 'Default' ATTRIBUTE_EXTENSIBLE = 'Extensible' +ATTRIBUTE_NO_INTERRUPT = 'NoInterrupt' ATTRIBUTE_STABLE = 'Stable' ATTRIBUTE_SYNC = 'Sync' +ATTRIBUTE_UNLIMITED_SIZE = 'UnlimitedSize' +ATTRIBUTE_UUID = 'Uuid' class NamedValue(object): @@ -274,6 +312,9 @@ class NamedValue(object): and (self.parent_kind, self.mojom_name) == (rhs.parent_kind, rhs.mojom_name)) + def __hash__(self): + return hash((self.parent_kind, self.mojom_name)) + class BuiltinValue(object): def __init__(self, value): @@ -368,21 +409,19 @@ class Field(object): class StructField(Field): - pass + def __hash__(self): + return super(Field, self).__hash__() class UnionField(Field): pass -def _IsFieldBackwardCompatible(new_field, old_field): +def _IsFieldBackwardCompatible(new_field, old_field, checker): if (new_field.min_version or 0) != (old_field.min_version or 0): return False - if isinstance(new_field.kind, (Enum, Struct, Union)): - return new_field.kind.IsBackwardCompatible(old_field.kind) - - return new_field.kind == old_field.kind + return checker.IsBackwardCompatible(new_field.kind, old_field.kind) class Struct(ReferenceKind): @@ -457,7 +496,7 @@ class Struct(ReferenceKind): for constant in self.constants: constant.Stylize(stylizer) - def IsBackwardCompatible(self, older_struct): + def IsBackwardCompatible(self, older_struct, checker): """This struct is backward-compatible with older_struct if and only if all of the following conditions hold: - Any newly added field is tagged with a [MinVersion] attribute specifying @@ -496,7 +535,7 @@ class Struct(ReferenceKind): old_field = old_fields[ordinal] if (old_field.min_version or 0) > max_old_min_version: max_old_min_version = old_field.min_version - if not _IsFieldBackwardCompatible(new_field, old_field): + if not _IsFieldBackwardCompatible(new_field, old_field, checker): # Type or min-version mismatch between old and new versions of the same # ordinal field. return False @@ -590,7 +629,7 @@ class Union(ReferenceKind): for field in self.fields: field.Stylize(stylizer) - def IsBackwardCompatible(self, older_union): + def IsBackwardCompatible(self, older_union, checker): """This union is backward-compatible with older_union if and only if all of the following conditions hold: - Any newly added field is tagged with a [MinVersion] attribute specifying @@ -623,7 +662,7 @@ class Union(ReferenceKind): if not new_field: # A field was removed, which is not OK. return False - if not _IsFieldBackwardCompatible(new_field, old_field): + if not _IsFieldBackwardCompatible(new_field, old_field, checker): # An field changed its type or MinVersion, which is not OK. return False old_min_version = old_field.min_version or 0 @@ -703,6 +742,10 @@ class Array(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return (isinstance(rhs, Array) and self.length == rhs.length + and checker.IsBackwardCompatible(self.kind, rhs.kind)) + class Map(ReferenceKind): """A map. @@ -747,6 +790,11 @@ class Map(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return (isinstance(rhs, Map) + and checker.IsBackwardCompatible(self.key_kind, rhs.key_kind) + and checker.IsBackwardCompatible(self.value_kind, rhs.value_kind)) + class PendingRemote(ReferenceKind): ReferenceKind.AddSharedProperty('kind') @@ -768,6 +816,10 @@ class PendingRemote(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return (isinstance(rhs, PendingRemote) + and checker.IsBackwardCompatible(self.kind, rhs.kind)) + class PendingReceiver(ReferenceKind): ReferenceKind.AddSharedProperty('kind') @@ -789,6 +841,10 @@ class PendingReceiver(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return isinstance(rhs, PendingReceiver) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + class PendingAssociatedRemote(ReferenceKind): ReferenceKind.AddSharedProperty('kind') @@ -810,6 +866,11 @@ class PendingAssociatedRemote(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return isinstance(rhs, + PendingAssociatedRemote) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + class PendingAssociatedReceiver(ReferenceKind): ReferenceKind.AddSharedProperty('kind') @@ -831,6 +892,11 @@ class PendingAssociatedReceiver(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return isinstance( + rhs, PendingAssociatedReceiver) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + class InterfaceRequest(ReferenceKind): ReferenceKind.AddSharedProperty('kind') @@ -851,6 +917,10 @@ class InterfaceRequest(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return isinstance(rhs, InterfaceRequest) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + class AssociatedInterfaceRequest(ReferenceKind): ReferenceKind.AddSharedProperty('kind') @@ -873,6 +943,11 @@ class AssociatedInterfaceRequest(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return isinstance( + rhs, AssociatedInterfaceRequest) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + class Parameter(object): def __init__(self, @@ -976,6 +1051,16 @@ class Method(object): return self.attributes.get(ATTRIBUTE_SYNC) \ if self.attributes else None + @property + def allow_interrupt(self): + return not self.attributes.get(ATTRIBUTE_NO_INTERRUPT) \ + if self.attributes else True + + @property + def unlimited_message_size(self): + return self.attributes.get(ATTRIBUTE_UNLIMITED_SIZE) \ + if self.attributes else False + def __eq__(self, rhs): return (isinstance(rhs, Method) and (self.mojom_name, self.ordinal, self.parameters, @@ -1029,7 +1114,7 @@ class Interface(ReferenceKind): for constant in self.constants: constant.Stylize(stylizer) - def IsBackwardCompatible(self, older_interface): + def IsBackwardCompatible(self, older_interface, checker): """This interface is backward-compatible with older_interface if and only if all of the following conditions hold: - All defined methods in older_interface (when identified by ordinal) have @@ -1067,8 +1152,8 @@ class Interface(ReferenceKind): # A method was removed, which is not OK. return False - if not new_method.param_struct.IsBackwardCompatible( - old_method.param_struct): + if not checker.IsBackwardCompatible(new_method.param_struct, + old_method.param_struct): # The parameter list is not backward-compatible, which is not OK. return False @@ -1081,8 +1166,8 @@ class Interface(ReferenceKind): if new_method.response_param_struct is None: # A reply was removed from a message, which is not OK. return False - if not new_method.response_param_struct.IsBackwardCompatible( - old_method.response_param_struct): + if not checker.IsBackwardCompatible(new_method.response_param_struct, + old_method.response_param_struct): # The new message's reply is not backward-compatible with the old # message's reply, which is not OK. return False @@ -1120,6 +1205,20 @@ class Interface(ReferenceKind): self.attributes) == (rhs.mojom_name, rhs.methods, rhs.enums, rhs.constants, rhs.attributes)) + @property + def uuid(self): + uuid_str = self.attributes.get(ATTRIBUTE_UUID) if self.attributes else None + if uuid_str is None: + return None + + try: + u = UUID(uuid_str) + except: + raise ValueError('Invalid format for Uuid attribute on interface {}. ' + 'Expected standard RFC 4122 string representation of ' + 'a UUID.'.format(self.mojom_name)) + return (int(u.hex[:16], 16), int(u.hex[16:], 16)) + def __hash__(self): return id(self) @@ -1144,6 +1243,11 @@ class AssociatedInterface(ReferenceKind): def __hash__(self): return id(self) + def IsBackwardCompatible(self, rhs, checker): + return isinstance(rhs, + AssociatedInterface) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + class EnumField(object): def __init__(self, @@ -1161,6 +1265,11 @@ class EnumField(object): self.name = stylizer.StylizeEnumField(self.mojom_name) @property + def default(self): + return self.attributes.get(ATTRIBUTE_DEFAULT, False) \ + if self.attributes else False + + @property def min_version(self): return self.attributes.get(ATTRIBUTE_MIN_VERSION) \ if self.attributes else None @@ -1186,6 +1295,7 @@ class Enum(Kind): self.attributes = attributes self.min_value = None self.max_value = None + self.default_field = None def Repr(self, as_ref=True): if as_ref: @@ -1216,7 +1326,8 @@ class Enum(Kind): prefix = self.module.GetNamespacePrefix() return '%s%s' % (prefix, self.mojom_name) - def IsBackwardCompatible(self, older_enum): + # pylint: disable=unused-argument + def IsBackwardCompatible(self, older_enum, checker): """This enum is backward-compatible with older_enum if and only if one of the following conditions holds: - Neither enum is [Extensible] and both have the exact same set of valid @@ -1250,9 +1361,10 @@ class Enum(Kind): def __eq__(self, rhs): return (isinstance(rhs, Enum) and (self.mojom_name, self.native_only, self.fields, self.attributes, - self.min_value, - self.max_value) == (rhs.mojom_name, rhs.native_only, rhs.fields, - rhs.attributes, rhs.min_value, rhs.max_value)) + self.min_value, self.max_value, + self.default_field) == (rhs.mojom_name, rhs.native_only, + rhs.fields, rhs.attributes, rhs.min_value, + rhs.max_value, rhs.default_field)) def __hash__(self): return id(self) @@ -1272,6 +1384,7 @@ class Module(object): self.attributes = attributes self.imports = [] self.imported_kinds = {} + self.metadata = {} def __repr__(self): # Gives us a decent __repr__ for modules. @@ -1285,6 +1398,9 @@ class Module(object): rhs.imports, rhs.constants, rhs.enums, rhs.structs, rhs.unions, rhs.interfaces)) + def __hash__(self): + return id(self) + def Repr(self, as_ref=True): if as_ref: return '<%s path=%r mojom_namespace=%r>' % ( @@ -1555,6 +1671,13 @@ def HasSyncMethods(interface): return False +def HasUninterruptableMethods(interface): + for method in interface.methods: + if not method.allow_interrupt: + return True + return False + + def ContainsHandlesOrInterfaces(kind): """Check if the kind contains any handles. diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py b/utils/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py index 7a300560..0da90058 100644 --- a/utils/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py +++ b/utils/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py @@ -75,9 +75,8 @@ def PrecompileTemplates(generator_modules, output_dir): os.path.dirname(module.__file__), generator.GetTemplatePrefix()) ])) jinja_env.filters.update(generator.GetFilters()) - jinja_env.compile_templates( - os.path.join(output_dir, "%s.zip" % generator.GetTemplatePrefix()), - extensions=["tmpl"], - zip="stored", - py_compile=True, - ignore_errors=False) + jinja_env.compile_templates(os.path.join( + output_dir, "%s.zip" % generator.GetTemplatePrefix()), + extensions=["tmpl"], + zip="stored", + ignore_errors=False) diff --git a/utils/ipc/mojo/public/tools/mojom/mojom/generate/translate.py b/utils/ipc/mojo/public/tools/mojom/mojom/generate/translate.py index d6df3ca6..7580b780 100644 --- a/utils/ipc/mojo/public/tools/mojom/mojom/generate/translate.py +++ b/utils/ipc/mojo/public/tools/mojom/mojom/generate/translate.py @@ -472,6 +472,9 @@ def _Method(module, parsed_method, interface): "attribute. If no response parameters are needed, you " "could use an empty response parameter list, i.e., " "\"=> ()\".") + # And only methods with the [Sync] attribute can specify [NoInterrupt]. + if not method.allow_interrupt and not method.sync: + raise Exception("Only [Sync] methods can be marked [NoInterrupt].") return method @@ -592,6 +595,16 @@ def _Enum(module, parsed_enum, parent_kind): map(lambda field: _EnumField(module, enum, field), parsed_enum.enum_value_list)) _ResolveNumericEnumValues(enum) + # TODO(https://crbug.com/731893): Require a default value to be + # specified. + for field in enum.fields: + if field.default: + if not enum.extensible: + raise Exception('Non-extensible enums may not specify a default') + if enum.default_field is not None: + raise Exception( + 'Only one enumerator value may be specified as the default') + enum.default_field = field module.kinds[enum.spec] = enum @@ -650,7 +663,9 @@ def _CollectReferencedKinds(module, all_defined_kinds): if mojom.IsMapKind(kind): return (extract_referenced_user_kinds(kind.key_kind) + extract_referenced_user_kinds(kind.value_kind)) - if mojom.IsInterfaceRequestKind(kind) or mojom.IsAssociatedKind(kind): + if (mojom.IsInterfaceRequestKind(kind) or mojom.IsAssociatedKind(kind) + or mojom.IsPendingRemoteKind(kind) + or mojom.IsPendingReceiverKind(kind)): return [kind.kind] if mojom.IsStructKind(kind): return [kind] @@ -678,12 +693,9 @@ def _CollectReferencedKinds(module, all_defined_kinds): for method in interface.methods: for param in itertools.chain(method.parameters or [], method.response_parameters or []): - if (mojom.IsStructKind(param.kind) or mojom.IsUnionKind(param.kind) - or mojom.IsEnumKind(param.kind) - or mojom.IsAnyInterfaceKind(param.kind)): - for referenced_kind in extract_referenced_user_kinds(param.kind): - sanitized_kind = sanitize_kind(referenced_kind) - referenced_user_kinds[sanitized_kind.spec] = sanitized_kind + for referenced_kind in extract_referenced_user_kinds(param.kind): + sanitized_kind = sanitize_kind(referenced_kind) + referenced_user_kinds[sanitized_kind.spec] = sanitized_kind return referenced_user_kinds diff --git a/utils/ipc/mojo/public/tools/mojom/mojom_parser.py b/utils/ipc/mojo/public/tools/mojom/mojom_parser.py index 12adbfb9..eb90c825 100755 --- a/utils/ipc/mojo/public/tools/mojom/mojom_parser.py +++ b/utils/ipc/mojo/public/tools/mojom/mojom_parser.py @@ -14,6 +14,8 @@ import argparse import codecs import errno import json +import logging +import multiprocessing import os import os.path import sys @@ -25,6 +27,19 @@ from mojom.parse import parser from mojom.parse import conditional_features +# Disable this for easier debugging. +# In Python 2, subprocesses just hang when exceptions are thrown :(. +_ENABLE_MULTIPROCESSING = sys.version_info[0] > 2 + +if sys.version_info < (3, 4): + _MULTIPROCESSING_USES_FORK = sys.platform.startswith('linux') +else: + # https://docs.python.org/3/library/multiprocessing.html#:~:text=bpo-33725 + if __name__ == '__main__' and sys.platform == 'darwin': + multiprocessing.set_start_method('fork') + _MULTIPROCESSING_USES_FORK = multiprocessing.get_start_method() == 'fork' + + def _ResolveRelativeImportPath(path, roots): """Attempts to resolve a relative import path against a set of possible roots. @@ -98,7 +113,7 @@ def _GetModuleFilename(mojom_filename): def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts, - dependencies, loaded_modules): + dependencies, loaded_modules, module_metadata): """Recursively ensures that a module and its dependencies are loaded. Args: @@ -111,10 +126,8 @@ def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts, by absolute file path. loaded_modules: A mapping of all modules loaded so far, including non-input modules that were pulled in as transitive dependencies of the inputs. - import_set: The working set of mojom imports processed so far in this - call stack. Used to detect circular dependencies. - import_stack: An ordered list of imports processed so far in this call - stack. Used to report circular dependencies. + module_metadata: Metadata to be attached to every module loaded by this + helper. Returns: None @@ -129,7 +142,7 @@ def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts, for dep_abspath, dep_path in dependencies[mojom_abspath]: if dep_abspath not in loaded_modules: _EnsureInputLoaded(dep_abspath, dep_path, abs_paths, asts, dependencies, - loaded_modules) + loaded_modules, module_metadata) imports = {} for imp in asts[mojom_abspath].import_list: @@ -137,6 +150,7 @@ def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts, imports[path] = loaded_modules[abs_paths[path]] loaded_modules[mojom_abspath] = translate.OrderedModule( asts[mojom_abspath], module_path, imports) + loaded_modules[mojom_abspath].metadata = dict(module_metadata) def _CollectAllowedImportsFromBuildMetadata(build_metadata_filename): @@ -157,10 +171,67 @@ def _CollectAllowedImportsFromBuildMetadata(build_metadata_filename): return allowed_imports +# multiprocessing helper. +def _ParseAstHelper(args): + mojom_abspath, enabled_features = args + with codecs.open(mojom_abspath, encoding='utf-8') as f: + ast = parser.Parse(f.read(), mojom_abspath) + conditional_features.RemoveDisabledDefinitions(ast, enabled_features) + return mojom_abspath, ast + + +# multiprocessing helper. +def _SerializeHelper(args): + mojom_abspath, mojom_path = args + module_path = os.path.join(_SerializeHelper.output_root_path, + _GetModuleFilename(mojom_path)) + module_dir = os.path.dirname(module_path) + if not os.path.exists(module_dir): + try: + # Python 2 doesn't support exist_ok on makedirs(), so we just ignore + # that failure if it happens. It's possible during build due to races + # among build steps with module outputs in the same directory. + os.makedirs(module_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + with open(module_path, 'wb') as f: + _SerializeHelper.loaded_modules[mojom_abspath].Dump(f) + + +def _Shard(target_func, args, processes=None): + args = list(args) + if processes is None: + processes = multiprocessing.cpu_count() + # Seems optimal to have each process perform at least 2 tasks. + processes = min(processes, len(args) // 2) + + if sys.platform == 'win32': + # TODO(crbug.com/1190269) - we can't use more than 56 + # cores on Windows or Python3 may hang. + processes = min(processes, 56) + + # Don't spin up processes unless there is enough work to merit doing so. + if not _ENABLE_MULTIPROCESSING or processes < 2: + for result in map(target_func, args): + yield result + return + + pool = multiprocessing.Pool(processes=processes) + try: + for result in pool.imap_unordered(target_func, args): + yield result + finally: + pool.close() + pool.join() # Needed on Windows to avoid WindowsError during terminate. + pool.terminate() + + def _ParseMojoms(mojom_files, input_root_paths, output_root_path, enabled_features, + module_metadata, allowed_imports=None): """Parses a set of mojom files and produces serialized module outputs. @@ -176,6 +247,8 @@ def _ParseMojoms(mojom_files, modules for any transitive dependencies not listed in mojom_files. enabled_features: A list of enabled feature names, controlling which AST nodes are filtered by [EnableIf] attributes. + module_metadata: A list of 2-tuples representing metadata key-value pairs to + attach to each compiled module output. Returns: None. @@ -193,72 +266,79 @@ def _ParseMojoms(mojom_files, for abs_path in mojom_files) abs_paths = dict( (path, abs_path) for abs_path, path in mojom_files_to_parse.items()) - for mojom_abspath, _ in mojom_files_to_parse.items(): - with codecs.open(mojom_abspath, encoding='utf-8') as f: - ast = parser.Parse(''.join(f.readlines()), mojom_abspath) - conditional_features.RemoveDisabledDefinitions(ast, enabled_features) - loaded_mojom_asts[mojom_abspath] = ast - invalid_imports = [] - for imp in ast.import_list: - import_abspath = _ResolveRelativeImportPath(imp.import_filename, - input_root_paths) - if allowed_imports and import_abspath not in allowed_imports: - invalid_imports.append(imp.import_filename) - - abs_paths[imp.import_filename] = import_abspath - if import_abspath in mojom_files_to_parse: - # This import is in the input list, so we're going to translate it - # into a module below; however it's also a dependency of another input - # module. We retain record of dependencies to help with input - # processing later. - input_dependencies[mojom_abspath].add((import_abspath, - imp.import_filename)) - else: - # We have an import that isn't being parsed right now. It must already - # be parsed and have a module file sitting in a corresponding output - # location. - module_path = _GetModuleFilename(imp.import_filename) - module_abspath = _ResolveRelativeImportPath(module_path, - [output_root_path]) - with open(module_abspath, 'rb') as module_file: - loaded_modules[import_abspath] = module.Module.Load(module_file) - - if invalid_imports: - raise ValueError( - '\nThe file %s imports the following files not allowed by build ' - 'dependencies:\n\n%s\n' % (mojom_abspath, - '\n'.join(invalid_imports))) + logging.info('Parsing %d .mojom into ASTs', len(mojom_files_to_parse)) + map_args = ((mojom_abspath, enabled_features) + for mojom_abspath in mojom_files_to_parse) + for mojom_abspath, ast in _Shard(_ParseAstHelper, map_args): + loaded_mojom_asts[mojom_abspath] = ast + + logging.info('Processing dependencies') + for mojom_abspath, ast in loaded_mojom_asts.items(): + invalid_imports = [] + for imp in ast.import_list: + import_abspath = _ResolveRelativeImportPath(imp.import_filename, + input_root_paths) + if allowed_imports and import_abspath not in allowed_imports: + invalid_imports.append(imp.import_filename) + + abs_paths[imp.import_filename] = import_abspath + if import_abspath in mojom_files_to_parse: + # This import is in the input list, so we're going to translate it + # into a module below; however it's also a dependency of another input + # module. We retain record of dependencies to help with input + # processing later. + input_dependencies[mojom_abspath].add( + (import_abspath, imp.import_filename)) + elif import_abspath not in loaded_modules: + # We have an import that isn't being parsed right now. It must already + # be parsed and have a module file sitting in a corresponding output + # location. + module_path = _GetModuleFilename(imp.import_filename) + module_abspath = _ResolveRelativeImportPath(module_path, + [output_root_path]) + with open(module_abspath, 'rb') as module_file: + loaded_modules[import_abspath] = module.Module.Load(module_file) + + if invalid_imports: + raise ValueError( + '\nThe file %s imports the following files not allowed by build ' + 'dependencies:\n\n%s\n' % (mojom_abspath, '\n'.join(invalid_imports))) + logging.info('Loaded %d modules from dependencies', len(loaded_modules)) # At this point all transitive imports not listed as inputs have been loaded # and we have a complete dependency tree of the unprocessed inputs. Now we can # load all the inputs, resolving dependencies among them recursively as we go. + logging.info('Ensuring inputs are loaded') num_existing_modules_loaded = len(loaded_modules) for mojom_abspath, mojom_path in mojom_files_to_parse.items(): _EnsureInputLoaded(mojom_abspath, mojom_path, abs_paths, loaded_mojom_asts, - input_dependencies, loaded_modules) + input_dependencies, loaded_modules, module_metadata) assert (num_existing_modules_loaded + len(mojom_files_to_parse) == len(loaded_modules)) # Now we have fully translated modules for every input and every transitive # dependency. We can dump the modules to disk for other tools to use. - for mojom_abspath, mojom_path in mojom_files_to_parse.items(): - module_path = os.path.join(output_root_path, _GetModuleFilename(mojom_path)) - module_dir = os.path.dirname(module_path) - if not os.path.exists(module_dir): - try: - # Python 2 doesn't support exist_ok on makedirs(), so we just ignore - # that failure if it happens. It's possible during build due to races - # among build steps with module outputs in the same directory. - os.makedirs(module_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - with open(module_path, 'wb') as f: - loaded_modules[mojom_abspath].Dump(f) + logging.info('Serializing %d modules', len(mojom_files_to_parse)) + + # Windows does not use fork() for multiprocessing, so we'd need to pass + # loaded_module via IPC rather than via globals. Doing so is slower than not + # using multiprocessing. + _SerializeHelper.loaded_modules = loaded_modules + _SerializeHelper.output_root_path = output_root_path + # Doesn't seem to help past 4. Perhaps IO bound here? + processes = 4 if _MULTIPROCESSING_USES_FORK else 0 + map_args = mojom_files_to_parse.items() + for _ in _Shard(_SerializeHelper, map_args, processes=processes): + pass def Run(command_line): + debug_logging = os.environ.get('MOJOM_PARSER_DEBUG', '0') != '0' + logging.basicConfig(level=logging.DEBUG if debug_logging else logging.WARNING, + format='%(levelname).1s %(relativeCreated)6d %(message)s') + logging.info('Started (%s)', os.path.basename(sys.argv[0])) + arg_parser = argparse.ArgumentParser( description=""" Parses one or more mojom files and produces corresponding module outputs fully @@ -333,6 +413,16 @@ already present in the provided output root.""") 'build-time dependency checking for mojom imports, where each build ' 'metadata file corresponds to a build target in the dependency graph of ' 'a typical build system.') + arg_parser.add_argument( + '--add-module-metadata', + dest='module_metadata', + default=[], + action='append', + metavar='KEY=VALUE', + help='Adds a metadata key-value pair to the output module. This can be ' + 'used by build toolchains to augment parsed mojom modules with product-' + 'specific metadata for later extraction and use by custom bindings ' + 'generators.') args, _ = arg_parser.parse_known_args(command_line) if args.mojom_file_list: @@ -353,8 +443,14 @@ already present in the provided output root.""") else: allowed_imports = None + module_metadata = list( + map(lambda kvp: tuple(kvp.split('=')), args.module_metadata)) _ParseMojoms(mojom_files, input_roots, output_root, args.enabled_features, - allowed_imports) + module_metadata, allowed_imports) + logging.info('Finished') + # Exit without running GC, which can save multiple seconds due the large + # number of object created. + os._exit(0) if __name__ == '__main__': diff --git a/utils/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py b/utils/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py index a0ee150e..65db4dc9 100644 --- a/utils/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py +++ b/utils/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py @@ -2,6 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from mojom.generate import module from mojom_parser_test_case import MojomParserTestCase @@ -20,9 +21,11 @@ class VersionCompatibilityTest(MojomParserTestCase): self.assertEqual(set(old.keys()), set(new.keys()), 'Old and new test mojoms should use the same type names.') + checker = module.BackwardCompatibilityChecker() compatibility_map = {} for name in old.keys(): - compatibility_map[name] = new[name].IsBackwardCompatible(old[name]) + compatibility_map[name] = checker.IsBackwardCompatible( + new[name], old[name]) return compatibility_map def assertBackwardCompatible(self, old_mojom, new_mojom): @@ -234,6 +237,47 @@ class VersionCompatibilityTest(MojomParserTestCase): self.assertNotBackwardCompatible('union U { string a; };', 'union U { string? a; };') + def testFieldNestedTypeChanged(self): + """Changing the definition of a nested type within a field (such as an array + element or interface endpoint type) should only break backward-compatibility + if the changes to that type are not backward-compatible.""" + self.assertBackwardCompatible( + """\ + struct S { string a; }; + struct T { array<S> ss; }; + """, """\ + struct S { + string a; + [MinVersion=1] string? b; + }; + struct T { array<S> ss; }; + """) + self.assertBackwardCompatible( + """\ + interface F { Do(); }; + struct S { pending_receiver<F> r; }; + """, """\ + interface F { + Do(); + [MinVersion=1] Say(); + }; + struct S { pending_receiver<F> r; }; + """) + + def testRecursiveTypeChange(self): + """Recursive types do not break the compatibility checker.""" + self.assertBackwardCompatible( + """\ + struct S { + string a; + array<S> others; + };""", """\ + struct S { + string a; + array<S> others; + [MinVersion=1] string? b; + };""") + def testUnionFieldBecomingNonOptional(self): """Changing a field from optional to non-optional breaks backward-compatibility.""" diff --git a/utils/ipc/tools/README b/utils/ipc/tools/README new file mode 100644 index 00000000..d5c24fc3 --- /dev/null +++ b/utils/ipc/tools/README @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: CC0-1.0 + +Files in this directory are imported from 9c138d992bfc of Chromium. Do not +modify them manually. |