From 50c92cc7e2924009ecab3e4004448b01d687707c Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 8 Aug 2024 18:13:00 +0300 Subject: meson: Move all code generation scripts to utils/codegen/ We have multiple code generation scripts in utils/, mixed with other miscellaneous utilities, as well as a larger code base based on mojom in utils/ipc/. To make code sharing easier between the generator scripts, without creating a mess in the utils/ directory, move all the code generation code to utils/codegen/. Signed-off-by: Laurent Pinchart Reviewed-by: Daniel Scally Reviewed-by: Paul Elder --- utils/codegen/ipc/mojo/public/tools/mojom/BUILD.gn | 18 + .../codegen/ipc/mojo/public/tools/mojom/README.md | 14 + .../mojom/check_stable_mojom_compatibility.py | 204 ++ .../check_stable_mojom_compatibility_unittest.py | 339 ++++ .../ipc/mojo/public/tools/mojom/const_unittest.py | 90 + .../ipc/mojo/public/tools/mojom/enum_unittest.py | 120 ++ .../mojo/public/tools/mojom/feature_unittest.py | 84 + .../ipc/mojo/public/tools/mojom/mojom/BUILD.gn | 43 + .../ipc/mojo/public/tools/mojom/mojom/__init__.py | 0 .../ipc/mojo/public/tools/mojom/mojom/error.py | 28 + .../ipc/mojo/public/tools/mojom/mojom/fileutil.py | 44 + .../public/tools/mojom/mojom/fileutil_unittest.py | 37 + .../public/tools/mojom/mojom/generate/__init__.py | 0 .../public/tools/mojom/mojom/generate/check.py | 26 + .../public/tools/mojom/mojom/generate/generator.py | 328 ++++ .../mojom/mojom/generate/generator_unittest.py | 71 + .../public/tools/mojom/mojom/generate/module.py | 2059 ++++++++++++++++++++ .../tools/mojom/mojom/generate/module_unittest.py | 31 + .../mojo/public/tools/mojom/mojom/generate/pack.py | 367 ++++ .../tools/mojom/mojom/generate/pack_unittest.py | 253 +++ .../mojom/mojom/generate/template_expander.py | 82 + .../public/tools/mojom/mojom/generate/translate.py | 1258 ++++++++++++ .../mojom/mojom/generate/translate_unittest.py | 141 ++ .../public/tools/mojom/mojom/parse/__init__.py | 0 .../ipc/mojo/public/tools/mojom/mojom/parse/ast.py | 462 +++++ .../public/tools/mojom/mojom/parse/ast_unittest.py | 115 ++ .../mojom/mojom/parse/conditional_features.py | 83 + .../mojom/parse/conditional_features_unittest.py | 376 ++++ .../mojo/public/tools/mojom/mojom/parse/lexer.py | 249 +++ .../tools/mojom/mojom/parse/lexer_unittest.py | 194 ++ .../mojo/public/tools/mojom/mojom/parse/parser.py | 510 +++++ .../tools/mojom/mojom/parse/parser_unittest.py | 1375 +++++++++++++ .../ipc/mojo/public/tools/mojom/mojom_parser.py | 502 +++++ .../public/tools/mojom/mojom_parser_test_case.py | 73 + .../public/tools/mojom/mojom_parser_unittest.py | 186 ++ .../tools/mojom/stable_attribute_unittest.py | 127 ++ .../ipc/mojo/public/tools/mojom/union_unittest.py | 44 + .../tools/mojom/version_compatibility_unittest.py | 458 +++++ 38 files changed, 10391 insertions(+) create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/BUILD.gn create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/README.md create mode 100755 utils/codegen/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py create mode 100755 utils/codegen/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/const_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/enum_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/feature_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/BUILD.gn create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/__init__.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/error.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/fileutil.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/fileutil_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/__init__.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/check.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/generator.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/generator_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/module.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/module_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/pack.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/pack_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/translate.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/translate_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/__init__.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/ast.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/ast_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/conditional_features.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/conditional_features_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/lexer.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/lexer_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/parser.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/parser_unittest.py create mode 100755 utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser_test_case.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/stable_attribute_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/union_unittest.py create mode 100644 utils/codegen/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py (limited to 'utils/codegen/ipc/mojo/public/tools/mojom') diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/BUILD.gn b/utils/codegen/ipc/mojo/public/tools/mojom/BUILD.gn new file mode 100644 index 00000000..eafb95a1 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/BUILD.gn @@ -0,0 +1,18 @@ +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +group("tests") { + data = [ + "check_stable_mojom_compatibility_unittest.py", + "check_stable_mojom_compatibility.py", + "const_unittest.py", + "enum_unittest.py", + "feature_unittest.py", + "mojom_parser_test_case.py", + "mojom_parser_unittest.py", + "mojom_parser.py", + "stable_attribute_unittest.py", + "version_compatibility_unittest.py", + ] +} diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/README.md b/utils/codegen/ipc/mojo/public/tools/mojom/README.md new file mode 100644 index 00000000..e5d17ab0 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/README.md @@ -0,0 +1,14 @@ +# The Mojom Parser + +The Mojom format is an interface definition language (IDL) for describing +interprocess communication (IPC) messages and data types for use with the +low-level cross-platform +[Mojo IPC library](https://chromium.googlesource.com/chromium/src/+/main/mojo/public/c/system/README.md). + +This directory consists of a `mojom` Python module, its tests, and supporting +command-line tools. The Python module implements the parser used by the +command-line tools and exposes an API to help external bindings generators emit +useful code from the parser's outputs. + +TODO(https://crbug.com/1060464): Fill out this documentation once the library +and tools have stabilized. diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py b/utils/codegen/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py new file mode 100755 index 00000000..35cd1cfd --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +# Copyright 2020 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Verifies backward-compatibility of mojom type changes. + +Given a set of pre- and post-diff mojom file contents, and a root directory +for a project, this tool verifies that any changes to [Stable] mojom types are +backward-compatible with the previous version. + +This can be used e.g. by a presubmit check to prevent developers from making +breaking changes to stable mojoms.""" + +import argparse +import io +import json +import os +import os.path +import sys + +from mojom.generate import module +from mojom.generate import translate +from mojom.parse import parser + +# pylint: disable=raise-missing-from + + +class ParseError(Exception): + pass + + +def _ValidateDelta(root, delta): + """Parses all modified mojoms (including all transitive mojom dependencies, + even if unmodified) to perform backward-compatibility checks on any types + marked with the [Stable] attribute. + + Note that unlike the normal build-time parser in mojom_parser.py, this does + not produce or rely on cached module translations, but instead parses the full + transitive closure of a mojom's input dependencies all at once. + """ + + translate.is_running_backwards_compatibility_check_hack = True + + # First build a map of all files covered by the delta + affected_files = set() + old_files = {} + new_files = {} + for change in delta: + # TODO(crbug.com/953884): Use pathlib once we're migrated fully to Python 3. + filename = change['filename'].replace('\\', '/') + affected_files.add(filename) + if change['old']: + old_files[filename] = change['old'] + if change['new']: + new_files[filename] = change['new'] + + # Parse and translate all mojoms relevant to the delta, including transitive + # imports that weren't modified. + unmodified_modules = {} + + def parseMojom(mojom, file_overrides, override_modules): + if mojom in unmodified_modules or mojom in override_modules: + return + + contents = file_overrides.get(mojom) + if contents: + modules = override_modules + else: + modules = unmodified_modules + with io.open(os.path.join(root, mojom), encoding='utf-8') as f: + contents = f.read() + + try: + ast = parser.Parse(contents, mojom) + except Exception as e: + raise ParseError('encountered exception {0} while parsing {1}'.format( + e, mojom)) + + # Files which are generated at compile time can't be checked by this script + # (at the moment) since they may not exist in the output directory. + generated_files_to_skip = { + ('third_party/blink/public/mojom/runtime_feature_state/' + 'runtime_feature.mojom'), + ('third_party/blink/public/mojom/origin_trial_feature/' + 'origin_trial_feature.mojom'), + } + + ast.import_list.items = [ + x for x in ast.import_list.items + if x.import_filename not in generated_files_to_skip + ] + + for imp in ast.import_list: + if (not file_overrides.get(imp.import_filename) + and not os.path.exists(os.path.join(root, imp.import_filename))): + # Speculatively construct a path prefix to locate the import_filename + mojom_path = os.path.dirname(os.path.normpath(mojom)).split(os.sep) + test_prefix = '' + for path_component in mojom_path: + test_prefix = os.path.join(test_prefix, path_component) + test_import_filename = os.path.join(test_prefix, imp.import_filename) + if os.path.exists(os.path.join(root, test_import_filename)): + imp.import_filename = test_import_filename + break + parseMojom(imp.import_filename, file_overrides, override_modules) + + # Now that the transitive set of dependencies has been imported and parsed + # above, translate each mojom AST into a Module so that all types are fully + # defined and can be inspected. + all_modules = {} + all_modules.update(unmodified_modules) + all_modules.update(override_modules) + modules[mojom] = translate.OrderedModule(ast, mojom, all_modules) + + old_modules = {} + for mojom in old_files: + parseMojom(mojom, old_files, old_modules) + new_modules = {} + for mojom in new_files: + parseMojom(mojom, new_files, new_modules) + + # At this point we have a complete set of translated Modules from both the + # pre- and post-diff mojom contents. Now we can analyze backward-compatibility + # of the deltas. + # + # Note that for backward-compatibility checks we only care about types which + # were marked [Stable] before the diff. Types newly marked as [Stable] are not + # checked. + def collectTypes(modules): + types = {} + for m in modules.values(): + for kinds in (m.enums, m.structs, m.unions, m.interfaces): + for kind in kinds: + types[kind.qualified_name] = kind + return types + + old_types = collectTypes(old_modules) + new_types = collectTypes(new_modules) + + # Collect any renamed types so they can be compared accordingly. + renamed_types = {} + for name, kind in new_types.items(): + old_name = kind.attributes and kind.attributes.get('RenamedFrom') + if old_name: + renamed_types[old_name] = name + + for qualified_name, kind in old_types.items(): + if not kind.stable: + continue + + new_name = renamed_types.get(qualified_name, qualified_name) + if new_name not in new_types: + raise Exception( + 'Stable type %s appears to be deleted by this change. If it was ' + 'renamed, please add a [RenamedFrom] attribute to the new type. This ' + 'can be deleted by a subsequent change.' % qualified_name) + + checker = module.BackwardCompatibilityChecker() + try: + if not checker.IsBackwardCompatible(new_types[new_name], kind): + raise Exception( + 'Stable type %s appears to have changed in a way which ' + 'breaks backward-compatibility. Please fix!\n\nIf you ' + 'believe this assessment to be incorrect, please file a ' + 'Chromium bug against the "Internals>Mojo>Bindings" ' + 'component.' % qualified_name) + except Exception as e: + raise Exception( + 'Stable type %s appears to have changed in a way which ' + 'breaks backward-compatibility: \n\n%s.\nPlease fix!\n\nIf you ' + 'believe this assessment to be incorrect, please file a ' + 'Chromium bug against the "Internals>Mojo>Bindings" ' + 'component.' % (qualified_name, e)) + + +def Run(command_line, delta=None): + """Runs the tool with the given command_line. Normally this will read the + change description from stdin as a JSON-encoded list, but tests may pass a + delta directly for convenience.""" + arg_parser = argparse.ArgumentParser( + description='Verifies backward-compatibility of mojom type changes.', + epilog=""" +This tool reads a change description from stdin and verifies that all modified +[Stable] mojom types will retain backward-compatibility. The change description +must be a JSON-encoded list of objects, each with a "filename" key (path to a +changed mojom file, relative to ROOT); an "old" key whose value is a string of +the full file contents before the change, or null if the file is being added; +and a "new" key whose value is a string of the full file contents after the +change, or null if the file is being deleted.""") + arg_parser.add_argument( + '--src-root', + required=True, + action='store', + metavar='ROOT', + help='The root of the source tree in which the checked mojoms live.') + + args, _ = arg_parser.parse_known_args(command_line) + if not delta: + delta = json.load(sys.stdin) + _ValidateDelta(args.src_root, delta) + + +if __name__ == '__main__': + Run(sys.argv[1:]) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility_unittest.py new file mode 100755 index 00000000..06769c95 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/check_stable_mojom_compatibility_unittest.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +# Copyright 2020 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import json +import os +import os.path +import shutil +import tempfile +import unittest + +import check_stable_mojom_compatibility + +from mojom.generate import module + + +class Change: + """Helper to clearly define a mojom file delta to be analyzed.""" + + def __init__(self, filename, old=None, new=None): + """If old is None, this is a file addition. If new is None, this is a file + deletion. Otherwise it's a file change.""" + self.filename = filename + self.old = old + self.new = new + + +class UnchangedFile(Change): + def __init__(self, filename, contents): + super().__init__(filename, old=contents, new=contents) + + +class CheckStableMojomCompatibilityTest(unittest.TestCase): + """Tests covering the behavior of the compatibility checking tool. Note that + details of different compatibility checks and relevant failure modes are NOT + covered by these tests. Those are instead covered by unittests in + version_compatibility_unittest.py. Additionally, the tests which ensure a + given set of [Stable] mojom definitions are indeed plausibly stable (i.e. they + have no unstable dependencies) are covered by stable_attribute_unittest.py. + + These tests cover higher-level concerns of the compatibility checking tool, + like file or symbol, renames, changes spread over multiple files, etc.""" + + def verifyBackwardCompatibility(self, changes): + """Helper for implementing assertBackwardCompatible and + assertNotBackwardCompatible""" + + temp_dir = tempfile.mkdtemp() + for change in changes: + if change.old: + # Populate the old file on disk in our temporary fake source root + file_path = os.path.join(temp_dir, change.filename) + dir_path = os.path.dirname(file_path) + if not os.path.exists(dir_path): + os.makedirs(dir_path) + with open(file_path, 'w') as f: + f.write(change.old) + + delta = [] + for change in changes: + if change.old != change.new: + delta.append({ + 'filename': change.filename, + 'old': change.old, + 'new': change.new + }) + + try: + check_stable_mojom_compatibility.Run(['--src-root', temp_dir], + delta=delta) + finally: + shutil.rmtree(temp_dir) + + def assertBackwardCompatible(self, changes): + self.verifyBackwardCompatibility(changes) + + def assertNotBackwardCompatible(self, changes): + try: + self.verifyBackwardCompatibility(changes) + except Exception: + return + + raise Exception('Change unexpectedly passed a backward-compatibility check') + + def testBasicCompatibility(self): + """Minimal smoke test to verify acceptance of a simple valid change.""" + self.assertBackwardCompatible([ + Change('foo/foo.mojom', + old='[Stable] struct S {};', + new='[Stable] struct S { [MinVersion=1] int32 x; };') + ]) + + def testBasicIncompatibility(self): + """Minimal smoke test to verify rejection of a simple invalid change.""" + self.assertNotBackwardCompatible([ + Change('foo/foo.mojom', + old='[Stable] struct S {};', + new='[Stable] struct S { int32 x; };') + ]) + + def testIgnoreIfNotStable(self): + """We don't care about types not marked [Stable]""" + self.assertBackwardCompatible([ + Change('foo/foo.mojom', + old='struct S {};', + new='struct S { int32 x; };') + ]) + + def testRename(self): + """We can do checks for renamed types.""" + self.assertBackwardCompatible([ + Change('foo/foo.mojom', + old='[Stable] struct S {};', + new='[Stable, RenamedFrom="S"] struct T {};') + ]) + self.assertNotBackwardCompatible([ + Change('foo/foo.mojom', + old='[Stable] struct S {};', + new='[Stable, RenamedFrom="S"] struct T { int32 x; };') + ]) + self.assertBackwardCompatible([ + Change('foo/foo.mojom', + old='[Stable] struct S {};', + new="""\ + [Stable, RenamedFrom="S"] + struct T { [MinVersion=1] int32 x; }; + """) + ]) + + def testNewlyStable(self): + """We don't care about types newly marked as [Stable].""" + self.assertBackwardCompatible([ + Change('foo/foo.mojom', + old='struct S {};', + new='[Stable] struct S { int32 x; };') + ]) + + def testFileRename(self): + """Make sure we can still do compatibility checks after a file rename.""" + self.assertBackwardCompatible([ + Change('foo/foo.mojom', old='[Stable] struct S {};', new=None), + Change('bar/bar.mojom', + old=None, + new='[Stable] struct S { [MinVersion=1] int32 x; };') + ]) + self.assertNotBackwardCompatible([ + Change('foo/foo.mojom', old='[Stable] struct S {};', new=None), + Change('bar/bar.mojom', old=None, new='[Stable] struct S { int32 x; };') + ]) + + def testWithImport(self): + """Ensure that cross-module dependencies do not break the compatibility + checking tool.""" + self.assertBackwardCompatible([ + Change('foo/foo.mojom', + old="""\ + module foo; + [Stable] struct S {}; + """, + new="""\ + module foo; + [Stable] struct S { [MinVersion=2] int32 x; }; + """), + Change('bar/bar.mojom', + old="""\ + module bar; + import "foo/foo.mojom"; + [Stable] struct T { foo.S s; }; + """, + new="""\ + module bar; + import "foo/foo.mojom"; + [Stable] struct T { foo.S s; [MinVersion=1] int32 y; }; + """) + ]) + + def testWithMovedDefinition(self): + """If a definition moves from one file to another, we should still be able + to check compatibility accurately.""" + self.assertBackwardCompatible([ + Change('foo/foo.mojom', + old="""\ + module foo; + [Stable] struct S {}; + """, + new="""\ + module foo; + """), + Change('bar/bar.mojom', + old="""\ + module bar; + import "foo/foo.mojom"; + [Stable] struct T { foo.S s; }; + """, + new="""\ + module bar; + import "foo/foo.mojom"; + [Stable, RenamedFrom="foo.S"] struct S { + [MinVersion=2] int32 x; + }; + [Stable] struct T { S s; [MinVersion=1] int32 y; }; + """) + ]) + + self.assertNotBackwardCompatible([ + Change('foo/foo.mojom', + old="""\ + module foo; + [Stable] struct S {}; + """, + new="""\ + module foo; + """), + Change('bar/bar.mojom', + old="""\ + module bar; + import "foo/foo.mojom"; + [Stable] struct T { foo.S s; }; + """, + new="""\ + module bar; + import "foo/foo.mojom"; + [Stable, RenamedFrom="foo.S"] struct S { int32 x; }; + [Stable] struct T { S s; [MinVersion=1] int32 y; }; + """) + ]) + + def testWithUnmodifiedImport(self): + """Unchanged files in the filesystem are still parsed by the compatibility + checking tool if they're imported by a changed file.""" + self.assertBackwardCompatible([ + UnchangedFile('foo/foo.mojom', 'module foo; [Stable] struct S {};'), + Change('bar/bar.mojom', + old="""\ + module bar; + import "foo/foo.mojom"; + [Stable] struct T { foo.S s; }; + """, + new="""\ + module bar; + import "foo/foo.mojom"; + [Stable] struct T { foo.S s; [MinVersion=1] int32 x; }; + """) + ]) + + self.assertNotBackwardCompatible([ + UnchangedFile('foo/foo.mojom', 'module foo; [Stable] struct S {};'), + Change('bar/bar.mojom', + old="""\ + module bar; + import "foo/foo.mojom"; + [Stable] struct T { foo.S s; }; + """, + new="""\ + module bar; + import "foo/foo.mojom"; + [Stable] struct T { foo.S s; int32 x; }; + """) + ]) + + def testWithPartialImport(self): + """The compatibility checking tool correctly parses imports with partial + paths.""" + self.assertBackwardCompatible([ + UnchangedFile('foo/foo.mojom', 'module foo; [Stable] struct S {};'), + Change('foo/bar.mojom', + old="""\ + module bar; + import "foo/foo.mojom"; + [Stable] struct T { foo.S s; }; + """, + new="""\ + module bar; + import "foo.mojom"; + [Stable] struct T { foo.S s; }; + """) + ]) + + self.assertBackwardCompatible([ + UnchangedFile('foo/foo.mojom', 'module foo; [Stable] struct S {};'), + Change('foo/bar.mojom', + old="""\ + module bar; + import "foo.mojom"; + [Stable] struct T { foo.S s; }; + """, + new="""\ + module bar; + import "foo/foo.mojom"; + [Stable] struct T { foo.S s; }; + """) + ]) + + self.assertNotBackwardCompatible([ + UnchangedFile('foo/foo.mojom', 'module foo; [Stable] struct S {};'), + Change('bar/bar.mojom', + old="""\ + module bar; + import "foo/foo.mojom"; + [Stable] struct T { foo.S s; }; + """, + new="""\ + module bar; + import "foo.mojom"; + [Stable] struct T { foo.S s; }; + """) + ]) + + self.assertNotBackwardCompatible([ + UnchangedFile('foo/foo.mojom', 'module foo; [Stable] struct S {};'), + Change('bar/bar.mojom', + old="""\ + module bar; + import "foo.mojom"; + [Stable] struct T { foo.S s; }; + """, + new="""\ + module bar; + import "foo/foo.mojom"; + [Stable] struct T { foo.S s; }; + """) + ]) + + def testNewEnumDefault(self): + # Should be backwards compatible since it does not affect the wire format. + # This specific case also checks that the backwards compatibility checker + # does not throw an error due to the older version of the enum not + # specifying [Default]. + self.assertBackwardCompatible([ + Change('foo/foo.mojom', + old='[Extensible] enum E { One };', + new='[Extensible] enum E { [Default] One };') + ]) + self.assertBackwardCompatible([ + Change('foo/foo.mojom', + old='[Extensible] enum E { [Default] One, Two, };', + new='[Extensible] enum E { One, [Default] Two, };') + ]) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/const_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/const_unittest.py new file mode 100644 index 00000000..e8ed36a7 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/const_unittest.py @@ -0,0 +1,90 @@ +# Copyright 2020 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from mojom_parser_test_case import MojomParserTestCase +from mojom.generate import module as mojom + + +class ConstTest(MojomParserTestCase): + """Tests constant parsing behavior.""" + + def testLiteralInt(self): + a_mojom = 'a.mojom' + self.WriteFile(a_mojom, 'const int32 k = 42;') + self.ParseMojoms([a_mojom]) + a = self.LoadModule(a_mojom) + self.assertEqual(1, len(a.constants)) + self.assertEqual('k', a.constants[0].mojom_name) + self.assertEqual('42', a.constants[0].value) + + def testLiteralFloat(self): + a_mojom = 'a.mojom' + self.WriteFile(a_mojom, 'const float k = 42.5;') + self.ParseMojoms([a_mojom]) + a = self.LoadModule(a_mojom) + self.assertEqual(1, len(a.constants)) + self.assertEqual('k', a.constants[0].mojom_name) + self.assertEqual('42.5', a.constants[0].value) + + def testLiteralString(self): + a_mojom = 'a.mojom' + self.WriteFile(a_mojom, 'const string k = "woot";') + self.ParseMojoms([a_mojom]) + a = self.LoadModule(a_mojom) + self.assertEqual(1, len(a.constants)) + self.assertEqual('k', a.constants[0].mojom_name) + self.assertEqual('"woot"', a.constants[0].value) + + def testEnumConstant(self): + a_mojom = 'a.mojom' + self.WriteFile(a_mojom, 'module a; enum E { kA = 41, kB };') + b_mojom = 'b.mojom' + self.WriteFile( + b_mojom, """\ + import "a.mojom"; + const a.E kE1 = a.E.kB; + + // We also allow value names to be unqualified, implying scope from the + // constant's type. + const a.E kE2 = kB; + """) + self.ParseMojoms([a_mojom, b_mojom]) + a = self.LoadModule(a_mojom) + b = self.LoadModule(b_mojom) + self.assertEqual(1, len(a.enums)) + self.assertEqual('E', a.enums[0].mojom_name) + self.assertEqual(2, len(b.constants)) + self.assertEqual('kE1', b.constants[0].mojom_name) + self.assertEqual(a.enums[0], b.constants[0].kind) + self.assertEqual(a.enums[0].fields[1], b.constants[0].value.field) + self.assertEqual(42, b.constants[0].value.field.numeric_value) + self.assertEqual('kE2', b.constants[1].mojom_name) + self.assertEqual(a.enums[0].fields[1], b.constants[1].value.field) + self.assertEqual(42, b.constants[1].value.field.numeric_value) + + def testConstantReference(self): + a_mojom = 'a.mojom' + self.WriteFile(a_mojom, 'const int32 kA = 42; const int32 kB = kA;') + self.ParseMojoms([a_mojom]) + a = self.LoadModule(a_mojom) + self.assertEqual(2, len(a.constants)) + self.assertEqual('kA', a.constants[0].mojom_name) + self.assertEqual('42', a.constants[0].value) + self.assertEqual('kB', a.constants[1].mojom_name) + self.assertEqual('42', a.constants[1].value) + + def testImportedConstantReference(self): + a_mojom = 'a.mojom' + self.WriteFile(a_mojom, 'const int32 kA = 42;') + b_mojom = 'b.mojom' + self.WriteFile(b_mojom, 'import "a.mojom"; const int32 kB = kA;') + self.ParseMojoms([a_mojom, b_mojom]) + a = self.LoadModule(a_mojom) + b = self.LoadModule(b_mojom) + self.assertEqual(1, len(a.constants)) + self.assertEqual(1, len(b.constants)) + self.assertEqual('kA', a.constants[0].mojom_name) + self.assertEqual('42', a.constants[0].value) + self.assertEqual('kB', b.constants[0].mojom_name) + self.assertEqual('42', b.constants[0].value) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/enum_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/enum_unittest.py new file mode 100644 index 00000000..9269cde5 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/enum_unittest.py @@ -0,0 +1,120 @@ +# Copyright 2020 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from mojom_parser_test_case import MojomParserTestCase + + +class EnumTest(MojomParserTestCase): + """Tests enum parsing behavior.""" + + def testExplicitValues(self): + """Verifies basic parsing of assigned integral values.""" + types = self.ExtractTypes('enum E { kFoo=0, kBar=2, kBaz };') + self.assertEqual('kFoo', types['E'].fields[0].mojom_name) + self.assertEqual(0, types['E'].fields[0].numeric_value) + self.assertEqual('kBar', types['E'].fields[1].mojom_name) + self.assertEqual(2, types['E'].fields[1].numeric_value) + self.assertEqual('kBaz', types['E'].fields[2].mojom_name) + self.assertEqual(3, types['E'].fields[2].numeric_value) + + def testImplicitValues(self): + """Verifies basic automatic assignment of integral values at parse time.""" + types = self.ExtractTypes('enum E { kFoo, kBar, kBaz };') + self.assertEqual('kFoo', types['E'].fields[0].mojom_name) + self.assertEqual(0, types['E'].fields[0].numeric_value) + self.assertEqual('kBar', types['E'].fields[1].mojom_name) + self.assertEqual(1, types['E'].fields[1].numeric_value) + self.assertEqual('kBaz', types['E'].fields[2].mojom_name) + self.assertEqual(2, types['E'].fields[2].numeric_value) + + def testSameEnumReference(self): + """Verifies that an enum value can be assigned from the value of another + field within the same enum.""" + types = self.ExtractTypes('enum E { kA, kB, kFirst=kA };') + self.assertEqual('kA', types['E'].fields[0].mojom_name) + self.assertEqual(0, types['E'].fields[0].numeric_value) + self.assertEqual('kB', types['E'].fields[1].mojom_name) + self.assertEqual(1, types['E'].fields[1].numeric_value) + self.assertEqual('kFirst', types['E'].fields[2].mojom_name) + self.assertEqual(0, types['E'].fields[2].numeric_value) + + def testSameModuleOtherEnumReference(self): + """Verifies that an enum value can be assigned from the value of a field + in another enum within the same module.""" + types = self.ExtractTypes('enum E { kA, kB }; enum F { kA = E.kB };') + self.assertEqual(1, types['F'].fields[0].numeric_value) + + def testImportedEnumReference(self): + """Verifies that an enum value can be assigned from the value of a field + in another enum within a different module.""" + a_mojom = 'a.mojom' + self.WriteFile(a_mojom, 'module a; enum E { kFoo=42, kBar };') + b_mojom = 'b.mojom' + self.WriteFile(b_mojom, + 'module b; import "a.mojom"; enum F { kFoo = a.E.kBar };') + self.ParseMojoms([a_mojom, b_mojom]) + b = self.LoadModule(b_mojom) + + self.assertEqual('F', b.enums[0].mojom_name) + self.assertEqual('kFoo', b.enums[0].fields[0].mojom_name) + self.assertEqual(43, b.enums[0].fields[0].numeric_value) + + def testConstantReference(self): + """Verifies that an enum value can be assigned from the value of an + integral constant within the same module.""" + types = self.ExtractTypes('const int32 kFoo = 42; enum E { kA = kFoo };') + self.assertEqual(42, types['E'].fields[0].numeric_value) + + def testInvalidConstantReference(self): + """Verifies that enum values cannot be assigned from the value of + non-integral constants.""" + with self.assertRaisesRegexp(ValueError, 'not an integer'): + self.ExtractTypes('const float kFoo = 1.0; enum E { kA = kFoo };') + with self.assertRaisesRegexp(ValueError, 'not an integer'): + self.ExtractTypes('const double kFoo = 1.0; enum E { kA = kFoo };') + with self.assertRaisesRegexp(ValueError, 'not an integer'): + self.ExtractTypes('const string kFoo = "lol"; enum E { kA = kFoo };') + + def testImportedConstantReference(self): + """Verifies that an enum value can be assigned from the value of an integral + constant within an imported module.""" + a_mojom = 'a.mojom' + self.WriteFile(a_mojom, 'module a; const int32 kFoo = 37;') + b_mojom = 'b.mojom' + self.WriteFile(b_mojom, + 'module b; import "a.mojom"; enum F { kFoo = a.kFoo };') + self.ParseMojoms([a_mojom, b_mojom]) + b = self.LoadModule(b_mojom) + + self.assertEqual('F', b.enums[0].mojom_name) + self.assertEqual('kFoo', b.enums[0].fields[0].mojom_name) + self.assertEqual(37, b.enums[0].fields[0].numeric_value) + + def testEnumAttributesAreEnums(self): + """Verifies that enum values in attributes are really enum types.""" + a_mojom = 'a.mojom' + self.WriteFile(a_mojom, 'module a; enum E { kFoo, kBar };') + b_mojom = 'b.mojom' + self.WriteFile( + b_mojom, 'module b;' + 'import "a.mojom";' + '[MooCow=a.E.kFoo]' + 'interface Foo { Foo(); };') + self.ParseMojoms([a_mojom, b_mojom]) + b = self.LoadModule(b_mojom) + self.assertEqual(b.interfaces[0].attributes['MooCow'].mojom_name, 'kFoo') + + def testConstantAttributes(self): + """Verifies that constants as attributes are translated to the constant.""" + a_mojom = 'a.mojom' + self.WriteFile( + a_mojom, 'module a;' + 'enum E { kFoo, kBar };' + 'const E kB = E.kFoo;' + '[Attr=kB] interface Hello { Foo(); };') + self.ParseMojoms([a_mojom]) + a = self.LoadModule(a_mojom) + self.assertEqual(a.interfaces[0].attributes['Attr'].mojom_name, 'kB') + self.assertEquals(a.interfaces[0].attributes['Attr'].value.mojom_name, + 'kFoo') diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/feature_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/feature_unittest.py new file mode 100644 index 00000000..5f014e1c --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/feature_unittest.py @@ -0,0 +1,84 @@ +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from mojom_parser_test_case import MojomParserTestCase + + +class FeatureTest(MojomParserTestCase): + """Tests feature parsing behavior.""" + def testFeatureOff(self): + """Verifies basic parsing of feature types.""" + types = self.ExtractTypes(""" + // e.g. BASE_DECLARE_FEATURE(kFeature); + [AttributeOne=ValueOne] + feature kFeature { + // BASE_FEATURE(kFeature,"MyFeature", + // base::FEATURE_DISABLED_BY_DEFAULT); + const string name = "MyFeature"; + const bool default_state = false; + }; + """) + self.assertEqual('name', types['kFeature'].constants[0].mojom_name) + self.assertEqual('"MyFeature"', types['kFeature'].constants[0].value) + self.assertEqual('default_state', types['kFeature'].constants[1].mojom_name) + self.assertEqual('false', types['kFeature'].constants[1].value) + + def testFeatureOn(self): + """Verifies basic parsing of feature types.""" + types = self.ExtractTypes(""" + // e.g. BASE_DECLARE_FEATURE(kFeature); + feature kFeature { + // BASE_FEATURE(kFeature,"MyFeature", + // base::FEATURE_ENABLED_BY_DEFAULT); + const string name = "MyFeature"; + const bool default_state = true; + }; + """) + self.assertEqual('name', types['kFeature'].constants[0].mojom_name) + self.assertEqual('"MyFeature"', types['kFeature'].constants[0].value) + self.assertEqual('default_state', types['kFeature'].constants[1].mojom_name) + self.assertEqual('true', types['kFeature'].constants[1].value) + + def testFeatureWeakKeyword(self): + """Verifies that `feature` is a weak keyword.""" + types = self.ExtractTypes(""" + // e.g. BASE_DECLARE_FEATURE(kFeature); + [AttributeOne=ValueOne] + feature kFeature { + // BASE_FEATURE(kFeature,"MyFeature", + // base::FEATURE_DISABLED_BY_DEFAULT); + const string name = "MyFeature"; + const bool default_state = false; + }; + struct MyStruct { + bool feature = true; + }; + interface InterfaceName { + Method(string feature) => (int32 feature); + }; + """) + self.assertEqual('name', types['kFeature'].constants[0].mojom_name) + self.assertEqual('"MyFeature"', types['kFeature'].constants[0].value) + self.assertEqual('default_state', types['kFeature'].constants[1].mojom_name) + self.assertEqual('false', types['kFeature'].constants[1].value) + + def testFeatureAttributesAreFeatures(self): + """Verifies that feature values in attributes are really feature types.""" + a_mojom = 'a.mojom' + self.WriteFile( + a_mojom, 'module a;' + 'feature F { const string name = "f";' + 'const bool default_state = false; };') + b_mojom = 'b.mojom' + self.WriteFile( + b_mojom, 'module b;' + 'import "a.mojom";' + 'feature G' + '{const string name = "g"; const bool default_state = false;};' + '[Attri=a.F] interface Foo { Foo(); };' + '[Boink=G] interface Bar {};') + self.ParseMojoms([a_mojom, b_mojom]) + b = self.LoadModule(b_mojom) + self.assertEqual(b.interfaces[0].attributes['Attri'].mojom_name, 'F') + self.assertEqual(b.interfaces[1].attributes['Boink'].mojom_name, 'G') diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/BUILD.gn b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/BUILD.gn new file mode 100644 index 00000000..a0edf0eb --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/BUILD.gn @@ -0,0 +1,43 @@ +# Copyright 2020 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +group("mojom") { + data = [ + "__init__.py", + "error.py", + "fileutil.py", + "generate/__init__.py", + "generate/check.py", + "generate/generator.py", + "generate/module.py", + "generate/pack.py", + "generate/template_expander.py", + "generate/translate.py", + "parse/__init__.py", + "parse/ast.py", + "parse/conditional_features.py", + "parse/lexer.py", + "parse/parser.py", + + # Third-party module dependencies + "//third_party/jinja2/", + "//third_party/ply/", + ] +} + +group("tests") { + data = [ + "fileutil_unittest.py", + "generate/generator_unittest.py", + "generate/module_unittest.py", + "generate/pack_unittest.py", + "generate/translate_unittest.py", + "parse/ast_unittest.py", + "parse/conditional_features_unittest.py", + "parse/lexer_unittest.py", + "parse/parser_unittest.py", + ] + + public_deps = [ ":mojom" ] +} diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/__init__.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/error.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/error.py new file mode 100644 index 00000000..dd53b835 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/error.py @@ -0,0 +1,28 @@ +# Copyright 2014 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +class Error(Exception): + """Base class for Mojo IDL bindings parser/generator errors.""" + + def __init__(self, filename, message, lineno=None, addenda=None, **kwargs): + """|filename| is the (primary) file which caused the error, |message| is the + error message, |lineno| is the 1-based line number (or |None| if not + applicable/available), and |addenda| is a list of additional lines to append + to the final error message.""" + Exception.__init__(self, **kwargs) + self.filename = filename + self.message = message + self.lineno = lineno + self.addenda = addenda + + def __str__(self): + if self.lineno: + s = "%s:%d: Error: %s" % (self.filename, self.lineno, self.message) + else: + s = "%s: Error: %s" % (self.filename, self.message) + return "\n".join([s] + self.addenda) if self.addenda else s + + def __repr__(self): + return str(self) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/fileutil.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/fileutil.py new file mode 100644 index 00000000..124f12c1 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/fileutil.py @@ -0,0 +1,44 @@ +# Copyright 2015 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import errno +import os.path +import sys + + +def _GetDirAbove(dirname): + """Returns the directory "above" this file containing |dirname| (which must + also be "above" this file).""" + path = os.path.abspath(__file__) + while True: + path, tail = os.path.split(path) + if not tail: + return None + if tail == dirname: + return path + + +def EnsureDirectoryExists(path, always_try_to_create=False): + """A wrapper for os.makedirs that does not error if the directory already + exists. A different process could be racing to create this directory.""" + + if not os.path.exists(path) or always_try_to_create: + try: + os.makedirs(path) + except OSError as e: + # There may have been a race to create this directory. + if e.errno != errno.EEXIST: + raise + + +def AddLocalRepoThirdPartyDirToModulePath(): + """Helper function to find the top-level directory of this script's repository + assuming the script falls somewhere within a 'mojo' directory, and insert the + top-level 'third_party' directory early in the module search path. Used to + ensure that third-party dependencies provided within the repository itself + (e.g. Chromium sources include snapshots of jinja2 and ply) are preferred over + locally installed system library packages.""" + toplevel_dir = _GetDirAbove('mojo') + if toplevel_dir: + sys.path.insert(1, os.path.join(toplevel_dir, 'third_party')) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/fileutil_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/fileutil_unittest.py new file mode 100644 index 00000000..c93d2289 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/fileutil_unittest.py @@ -0,0 +1,37 @@ +# Copyright 2015 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os.path +import shutil +import tempfile +import unittest + +from mojom import fileutil + +class FileUtilTest(unittest.TestCase): + def testEnsureDirectoryExists(self): + """Test that EnsureDirectoryExists functions correctly.""" + + temp_dir = tempfile.mkdtemp() + try: + self.assertTrue(os.path.exists(temp_dir)) + + # Directory does not exist, yet. + full = os.path.join(temp_dir, "foo", "bar") + self.assertFalse(os.path.exists(full)) + + # Create the directory. + fileutil.EnsureDirectoryExists(full) + self.assertTrue(os.path.exists(full)) + + # Trying to create it again does not cause an error. + fileutil.EnsureDirectoryExists(full) + self.assertTrue(os.path.exists(full)) + + # Bypass check for directory existence to tickle error handling that + # occurs in response to a race. + fileutil.EnsureDirectoryExists(full, always_try_to_create=True) + self.assertTrue(os.path.exists(full)) + finally: + shutil.rmtree(temp_dir) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/__init__.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/check.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/check.py new file mode 100644 index 00000000..1efe2022 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/check.py @@ -0,0 +1,26 @@ +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Code shared by the various pre-generation mojom checkers.""" + + +class CheckException(Exception): + def __init__(self, module, message): + self.module = module + self.message = message + super().__init__(self.message) + + def __str__(self): + return "Failed mojo pre-generation check for {}:\n{}".format( + self.module.path, self.message) + + +class Check: + def __init__(self, module): + self.module = module + + def CheckModule(self): + """ Subclass should return True if its Checks pass, and throw an + exception otherwise. CheckModule will be called immediately before + mojom.generate.Generator.GenerateFiles()""" + raise NotImplementedError("Subclasses must override/implement this method") diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/generator.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/generator.py new file mode 100644 index 00000000..96fe3a2d --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/generator.py @@ -0,0 +1,328 @@ +# Copyright 2013 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Code shared by the various language-specific code generators.""" + +from __future__ import print_function + +from functools import partial +import os.path +import re + +from mojom import fileutil +from mojom.generate import module as mojom +from mojom.generate import pack + + +def ExpectedArraySize(kind): + if mojom.IsArrayKind(kind): + return kind.length + return None + + +def SplitCamelCase(identifier): + """Splits a camel-cased |identifier| and returns a list of lower-cased + strings. + """ + # Add underscores after uppercase letters when appropriate. An uppercase + # letter is considered the end of a word if it is followed by an upper and a + # lower. E.g. URLLoaderFactory -> URL_LoaderFactory + identifier = re.sub('([A-Z][0-9]*)(?=[A-Z][0-9]*[a-z])', r'\1_', identifier) + # Add underscores after lowercase letters when appropriate. A lowercase letter + # is considered the end of a word if it is followed by an upper. + # E.g. URLLoaderFactory -> URLLoader_Factory + identifier = re.sub('([a-z][0-9]*)(?=[A-Z])', r'\1_', identifier) + return [x.lower() for x in identifier.split('_')] + + +def ToCamel(identifier, lower_initial=False, digits_split=False, delimiter='_'): + """Splits |identifier| using |delimiter|, makes the first character of each + word uppercased (but makes the first character of the first word lowercased + if |lower_initial| is set to True), and joins the words. Please note that for + each word, all the characters except the first one are untouched. + """ + result = '' + capitalize_next = True + for i in range(len(identifier)): + if identifier[i] == delimiter: + capitalize_next = True + elif digits_split and identifier[i].isdigit(): + capitalize_next = True + result += identifier[i] + elif capitalize_next: + capitalize_next = False + result += identifier[i].upper() + else: + result += identifier[i] + + if lower_initial and result: + result = result[0].lower() + result[1:] + + return result + + +def _ToSnakeCase(identifier, upper=False): + """Splits camel-cased |identifier| into lower case words, removes the first + word if it's "k" and joins them using "_" e.g. for "URLLoaderFactory", returns + "URL_LOADER_FACTORY" if upper, otherwise "url_loader_factory". + """ + words = SplitCamelCase(identifier) + if words[0] == 'k' and len(words) > 1: + words = words[1:] + + # Variables cannot start with a digit + if (words[0][0].isdigit()): + words[0] = '_' + words[0] + + + if upper: + words = map(lambda x: x.upper(), words) + + return '_'.join(words) + + +def ToUpperSnakeCase(identifier): + """Splits camel-cased |identifier| into lower case words, removes the first + word if it's "k" and joins them using "_" e.g. for "URLLoaderFactory", returns + "URL_LOADER_FACTORY". + """ + return _ToSnakeCase(identifier, upper=True) + + +def ToLowerSnakeCase(identifier): + """Splits camel-cased |identifier| into lower case words, removes the first + word if it's "k" and joins them using "_" e.g. for "URLLoaderFactory", returns + "url_loader_factory". + """ + return _ToSnakeCase(identifier, upper=False) + + +class Stylizer: + """Stylizers specify naming rules to map mojom names to names in generated + code. For example, if you would like method_name in mojom to be mapped to + MethodName in the generated code, you need to define a subclass of Stylizer + and override StylizeMethod to do the conversion.""" + + def StylizeConstant(self, mojom_name): + return mojom_name + + def StylizeField(self, mojom_name): + return mojom_name + + def StylizeStruct(self, mojom_name): + return mojom_name + + def StylizeUnion(self, mojom_name): + return mojom_name + + def StylizeParameter(self, mojom_name): + return mojom_name + + def StylizeMethod(self, mojom_name): + return mojom_name + + def StylizeInterface(self, mojom_name): + return mojom_name + + def StylizeEnumField(self, mojom_name): + return mojom_name + + def StylizeEnum(self, mojom_name): + return mojom_name + + def StylizeFeature(self, mojom_name): + return mojom_name + + def StylizeModule(self, mojom_namespace): + return mojom_namespace + + +def WriteFile(contents, full_path): + # If |contents| is same with the file content, we skip updating. + if not isinstance(contents, bytes): + data = contents.encode('utf8') + else: + data = contents + + if os.path.isfile(full_path): + with open(full_path, 'rb') as destination_file: + if destination_file.read() == data: + return + + # Make sure the containing directory exists. + full_dir = os.path.dirname(full_path) + fileutil.EnsureDirectoryExists(full_dir) + + # Dump the data to disk. + with open(full_path, 'wb') as f: + f.write(data) + + +def AddComputedData(module): + """Adds computed data to the given module. The data is computed once and + used repeatedly in the generation process.""" + + def _AddStructComputedData(exported, struct): + struct.packed = pack.PackedStruct(struct) + struct.bytes = pack.GetByteLayout(struct.packed) + struct.versions = pack.GetVersionInfo(struct.packed) + struct.exported = exported + + def _AddInterfaceComputedData(interface): + interface.version = 0 + for method in interface.methods: + # this field is never scrambled + method.sequential_ordinal = method.ordinal + + if method.min_version is not None: + interface.version = max(interface.version, method.min_version) + + method.param_struct = _GetStructFromMethod(method) + if interface.stable: + method.param_struct.attributes[mojom.ATTRIBUTE_STABLE] = True + if method.explicit_ordinal is None: + raise Exception( + 'Stable interfaces must declare explicit method ordinals. The ' + 'method %s on stable interface %s does not declare an explicit ' + 'ordinal.' % (method.mojom_name, interface.qualified_name)) + interface.version = max(interface.version, + method.param_struct.versions[-1].version) + + if method.response_parameters is not None: + method.response_param_struct = _GetResponseStructFromMethod(method) + if interface.stable: + method.response_param_struct.attributes[mojom.ATTRIBUTE_STABLE] = True + interface.version = max( + interface.version, + method.response_param_struct.versions[-1].version) + else: + method.response_param_struct = None + + def _GetStructFromMethod(method): + """Converts a method's parameters into the fields of a struct.""" + params_class = "%s_%s_Params" % (method.interface.mojom_name, + method.mojom_name) + struct = mojom.Struct(params_class, + module=method.interface.module, + attributes={}) + for param in method.parameters: + struct.AddField( + param.mojom_name, + param.kind, + param.ordinal, + attributes=param.attributes) + _AddStructComputedData(False, struct) + return struct + + def _GetResponseStructFromMethod(method): + """Converts a method's response_parameters into the fields of a struct.""" + params_class = "%s_%s_ResponseParams" % (method.interface.mojom_name, + method.mojom_name) + struct = mojom.Struct(params_class, + module=method.interface.module, + attributes={}) + for param in method.response_parameters: + struct.AddField( + param.mojom_name, + param.kind, + param.ordinal, + attributes=param.attributes) + _AddStructComputedData(False, struct) + return struct + + for struct in module.structs: + _AddStructComputedData(True, struct) + for interface in module.interfaces: + _AddInterfaceComputedData(interface) + + +class Generator: + # Pass |output_dir| to emit files to disk. Omit |output_dir| to echo all + # files to stdout. + def __init__(self, + module, + output_dir=None, + typemap=None, + variant=None, + bytecode_path=None, + for_blink=False, + js_generate_struct_deserializers=False, + export_attribute=None, + export_header=None, + generate_non_variant_code=False, + support_lazy_serialization=False, + disallow_native_types=False, + disallow_interfaces=False, + generate_message_ids=False, + generate_fuzzing=False, + enable_kythe_annotations=False, + extra_cpp_template_paths=None, + generate_extra_cpp_only=False): + self.module = module + self.output_dir = output_dir + self.typemap = typemap or {} + self.variant = variant + self.bytecode_path = bytecode_path + self.for_blink = for_blink + self.js_generate_struct_deserializers = js_generate_struct_deserializers + self.export_attribute = export_attribute + self.export_header = export_header + self.generate_non_variant_code = generate_non_variant_code + self.support_lazy_serialization = support_lazy_serialization + self.disallow_native_types = disallow_native_types + self.disallow_interfaces = disallow_interfaces + self.generate_message_ids = generate_message_ids + self.generate_fuzzing = generate_fuzzing + self.enable_kythe_annotations = enable_kythe_annotations + self.extra_cpp_template_paths = extra_cpp_template_paths + self.generate_extra_cpp_only = generate_extra_cpp_only + + def Write(self, contents, filename): + if self.output_dir is None: + print(contents) + return + full_path = os.path.join(self.output_dir, filename) + WriteFile(contents, full_path) + + def OptimizeEmpty(self, contents): + # Look for .cc files that contain no actual code. There are many of these + # and they collectively take a while to compile. + lines = contents.splitlines() + + for line in lines: + if line.startswith('#') or line.startswith('//'): + continue + if re.match(r'namespace .* {', line) or re.match(r'}.*//.*namespace', + line): + continue + if line.strip(): + # There is some actual code - return the unmodified contents. + return contents + + # If we reach here then we have a .cc file with no actual code. The + # includes are therefore unneeded and can be removed. + new_lines = [line for line in lines if not line.startswith('#include')] + if len(new_lines) < len(lines): + new_lines.append('') + new_lines.append('// Includes removed due to no code being generated.') + return '\n'.join(new_lines) + + def WriteWithComment(self, contents, filename): + generator_name = "mojom_bindings_generator.py" + comment = r"// %s is auto generated by %s, do not edit" % (filename, + generator_name) + contents = comment + '\n' + '\n' + contents; + if filename.endswith('.cc'): + contents = self.OptimizeEmpty(contents) + self.Write(contents, filename) + + def GenerateFiles(self, args): + raise NotImplementedError("Subclasses must override/implement this method") + + def GetJinjaParameters(self): + """Returns default constructor parameters for the jinja environment.""" + return {} + + def GetGlobals(self): + """Returns global mappings for the template generation.""" + return {} diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/generator_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/generator_unittest.py new file mode 100644 index 00000000..7143e07c --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/generator_unittest.py @@ -0,0 +1,71 @@ +# Copyright 2014 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import importlib.util +import os.path +import sys +import unittest + +def _GetDirAbove(dirname): + """Returns the directory "above" this file containing |dirname| (which must + also be "above" this file).""" + path = os.path.abspath(__file__) + while True: + path, tail = os.path.split(path) + assert tail + if tail == dirname: + return path + + +try: + importlib.util.find_spec("mojom") +except ImportError: + sys.path.append(os.path.join(_GetDirAbove("pylib"), "pylib")) +from mojom.generate import generator + +class StringManipulationTest(unittest.TestCase): + """generator contains some string utilities, this tests only those.""" + + def testSplitCamelCase(self): + self.assertEquals(["camel", "case"], generator.SplitCamelCase("CamelCase")) + self.assertEquals(["url", "loader", "factory"], + generator.SplitCamelCase('URLLoaderFactory')) + self.assertEquals(["get99", "entries"], + generator.SplitCamelCase('Get99Entries')) + self.assertEquals(["get99entries"], + generator.SplitCamelCase('Get99entries')) + + def testToCamel(self): + self.assertEquals("CamelCase", generator.ToCamel("camel_case")) + self.assertEquals("CAMELCASE", generator.ToCamel("CAMEL_CASE")) + self.assertEquals("camelCase", + generator.ToCamel("camel_case", lower_initial=True)) + self.assertEquals("CamelCase", generator.ToCamel( + "camel case", delimiter=' ')) + self.assertEquals("CaMelCaSe", generator.ToCamel("caMel_caSe")) + self.assertEquals("L2Tp", generator.ToCamel("l2tp", digits_split=True)) + self.assertEquals("l2tp", generator.ToCamel("l2tp", lower_initial=True)) + + def testToSnakeCase(self): + self.assertEquals("snake_case", generator.ToLowerSnakeCase("SnakeCase")) + self.assertEquals("snake_case", generator.ToLowerSnakeCase("snakeCase")) + self.assertEquals("snake_case", generator.ToLowerSnakeCase("SnakeCASE")) + self.assertEquals("snake_d3d11_case", + generator.ToLowerSnakeCase("SnakeD3D11Case")) + self.assertEquals("snake_d3d11_case", + generator.ToLowerSnakeCase("SnakeD3d11Case")) + self.assertEquals("snake_d3d11_case", + generator.ToLowerSnakeCase("snakeD3d11Case")) + self.assertEquals("SNAKE_CASE", generator.ToUpperSnakeCase("SnakeCase")) + self.assertEquals("SNAKE_CASE", generator.ToUpperSnakeCase("snakeCase")) + self.assertEquals("SNAKE_CASE", generator.ToUpperSnakeCase("SnakeCASE")) + self.assertEquals("SNAKE_D3D11_CASE", + generator.ToUpperSnakeCase("SnakeD3D11Case")) + self.assertEquals("SNAKE_D3D11_CASE", + generator.ToUpperSnakeCase("SnakeD3d11Case")) + self.assertEquals("SNAKE_D3D11_CASE", + generator.ToUpperSnakeCase("snakeD3d11Case")) + +if __name__ == "__main__": + unittest.main() diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/module.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/module.py new file mode 100644 index 00000000..ca71059d --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/module.py @@ -0,0 +1,2059 @@ +# Copyright 2013 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This module's classes provide an interface to mojo modules. Modules are +# collections of interfaces and structs to be used by mojo ipc clients and +# servers. +# +# A simple interface would be created this way: +# module = mojom.generate.module.Module('Foo') +# interface = module.AddInterface('Bar') +# method = interface.AddMethod('Tat', 0) +# method.AddParameter('baz', 0, mojom.INT32) + +import pickle +from collections import OrderedDict +from uuid import UUID + +# pylint: disable=raise-missing-from + + +class BackwardCompatibilityChecker: + """Used for memoization while recursively checking two type definitions for + backward-compatibility.""" + + def __init__(self): + self._cache = {} + + def IsBackwardCompatible(self, new_kind, old_kind): + key = (new_kind, old_kind) + result = self._cache.get(key) + if result is None: + # Assume they're compatible at first to effectively ignore recursive + # checks between these types, e.g. if both kinds are a struct or union + # that references itself in a field. + self._cache[key] = True + result = new_kind.IsBackwardCompatible(old_kind, self) + self._cache[key] = result + return result + + +# We use our own version of __repr__ when displaying the AST, as the +# AST currently doesn't capture which nodes are reference (e.g. to +# types) and which nodes are definitions. This allows us to e.g. print +# the definition of a struct when it's defined inside a module, but +# only print its name when it's referenced in e.g. a method parameter. +def Repr(obj, as_ref=True): + """A version of __repr__ that can distinguish references. + + Sometimes we like to print an object's full representation + (e.g. with its fields) and sometimes we just want to reference an + object that was printed in full elsewhere. This function allows us + to make that distinction. + + Args: + obj: The object whose string representation we compute. + as_ref: If True, use the short reference representation. + + Returns: + A str representation of |obj|. + """ + if hasattr(obj, 'Repr'): + return obj.Repr(as_ref=as_ref) + # Since we cannot implement Repr for existing container types, we + # handle them here. + if isinstance(obj, list): + if not obj: + return '[]' + return ('[\n%s\n]' % + (',\n'.join(' %s' % Repr(elem, as_ref).replace('\n', '\n ') + for elem in obj))) + if isinstance(obj, dict): + if not obj: + return '{}' + return ('{\n%s\n}' % (',\n'.join(' %s: %s' % + (Repr(key, as_ref).replace('\n', '\n '), + Repr(val, as_ref).replace('\n', '\n ')) + for key, val in obj.items()))) + return repr(obj) + + +def GenericRepr(obj, names): + """Compute generic Repr for |obj| based on the attributes in |names|. + + Args: + obj: The object to compute a Repr for. + names: A dict from attribute names to include, to booleans + specifying whether those attributes should be shown as + references or not. + + Returns: + A str representation of |obj|. + """ + + def ReprIndent(name, as_ref): + return ' %s=%s' % (name, Repr(getattr(obj, name), as_ref).replace( + '\n', '\n ')) + + return '%s(\n%s\n)' % (obj.__class__.__name__, ',\n'.join( + ReprIndent(name, as_ref) for (name, as_ref) in names.items())) + + +class Kind: + """Kind represents a type (e.g. int8, string). + + Attributes: + spec: A string uniquely identifying the type. May be None. + module: {Module} The defining module. Set to None for built-in types. + parent_kind: The enclosing type. For example, an enum defined + inside an interface has that interface as its parent. May be None. + is_nullable: True if the type is nullable. + """ + + def __init__(self, spec=None, is_nullable=False, module=None): + self.spec = spec + self.module = module + self.parent_kind = None + self.is_nullable = is_nullable + self.shared_definition = {} + + @classmethod + def AddSharedProperty(cls, name): + """Adds a property |name| to |cls|, which accesses the corresponding item in + |shared_definition|. + + The reason of adding such indirection is to enable sharing definition + between a reference kind and its nullable variation. For example: + a = Struct('test_struct_1') + b = a.MakeNullableKind() + a.name = 'test_struct_2' + print(b.name) # Outputs 'test_struct_2'. + """ + def Get(self): + try: + return self.shared_definition[name] + except KeyError: # Must raise AttributeError if property doesn't exist. + raise AttributeError + + def Set(self, value): + self.shared_definition[name] = value + + setattr(cls, name, property(Get, Set)) + + def Repr(self, as_ref=True): + # pylint: disable=unused-argument + return '<%s spec=%r is_nullable=%r>' % (self.__class__.__name__, self.spec, + self.is_nullable) + + def __repr__(self): + # Gives us a decent __repr__ for all kinds. + return self.Repr() + + def __eq__(self, rhs): + # pylint: disable=unidiomatic-typecheck + return (type(self) == type(rhs) + and (self.spec, self.parent_kind, self.is_nullable) + == (rhs.spec, rhs.parent_kind, rhs.is_nullable)) + + def __hash__(self): + # TODO(crbug.com/1060471): Remove this and other __hash__ methods on Kind + # and its subclasses. This is to support existing generator code which uses + # some primitive Kinds as dict keys. The default hash (object identity) + # breaks these dicts when a pickled Module instance is unpickled and used + # during a subsequent run of the parser. + return hash((self.spec, self.parent_kind, self.is_nullable)) + + # pylint: disable=unused-argument + def IsBackwardCompatible(self, rhs, checker): + return self == rhs + + +class ValueKind(Kind): + """ValueKind represents values that aren't reference kinds. + + The primary difference is the wire representation for nullable value kinds + still reserves space for the value type itself, even if that value itself + is logically null. + """ + def __init__(self, spec=None, is_nullable=False, module=None): + assert spec is None or is_nullable == spec.startswith('?') + Kind.__init__(self, spec, is_nullable, module) + + def MakeNullableKind(self): + assert not self.is_nullable + + if self == BOOL: + return NULLABLE_BOOL + if self == INT8: + return NULLABLE_INT8 + if self == INT16: + return NULLABLE_INT16 + if self == INT32: + return NULLABLE_INT32 + if self == INT64: + return NULLABLE_INT64 + if self == UINT8: + return NULLABLE_UINT8 + if self == UINT16: + return NULLABLE_UINT16 + if self == UINT32: + return NULLABLE_UINT32 + if self == UINT64: + return NULLABLE_UINT64 + if self == FLOAT: + return NULLABLE_FLOAT + if self == DOUBLE: + return NULLABLE_DOUBLE + + nullable_kind = type(self)() + nullable_kind.shared_definition = self.shared_definition + if self.spec is not None: + nullable_kind.spec = '?' + self.spec + nullable_kind.is_nullable = True + nullable_kind.parent_kind = self.parent_kind + nullable_kind.module = self.module + + return nullable_kind + + def MakeUnnullableKind(self): + assert self.is_nullable + + if self == NULLABLE_BOOL: + return BOOL + if self == NULLABLE_INT8: + return INT8 + if self == NULLABLE_INT16: + return INT16 + if self == NULLABLE_INT32: + return INT32 + if self == NULLABLE_INT64: + return INT64 + if self == NULLABLE_UINT8: + return UINT8 + if self == NULLABLE_UINT16: + return UINT16 + if self == NULLABLE_UINT32: + return UINT32 + if self == NULLABLE_UINT64: + return UINT64 + if self == NULLABLE_FLOAT: + return FLOAT + if self == NULLABLE_DOUBLE: + return DOUBLE + + nullable_kind = type(self)() + nullable_kind.shared_definition = self.shared_definition + if self.spec is not None: + nullable_kind.spec = self.spec[1:] + nullable_kind.is_nullable = False + nullable_kind.parent_kind = self.parent_kind + nullable_kind.module = self.module + + return nullable_kind + + def __eq__(self, rhs): + return (isinstance(rhs, ValueKind) and super().__eq__(rhs)) + + def __hash__(self): # pylint: disable=useless-super-delegation + return super().__hash__() + + +class ReferenceKind(Kind): + """ReferenceKind represents pointer and handle types. + + A type is nullable if null (for pointer types) or invalid handle (for handle + types) is a legal value for the type. + """ + + def __init__(self, spec=None, is_nullable=False, module=None): + assert spec is None or is_nullable == spec.startswith('?') + Kind.__init__(self, spec, is_nullable, module) + + def MakeNullableKind(self): + assert not self.is_nullable + + if self == STRING: + return NULLABLE_STRING + if self == HANDLE: + return NULLABLE_HANDLE + if self == DCPIPE: + return NULLABLE_DCPIPE + if self == DPPIPE: + return NULLABLE_DPPIPE + if self == MSGPIPE: + return NULLABLE_MSGPIPE + if self == SHAREDBUFFER: + return NULLABLE_SHAREDBUFFER + if self == PLATFORMHANDLE: + return NULLABLE_PLATFORMHANDLE + + nullable_kind = type(self)() + nullable_kind.shared_definition = self.shared_definition + if self.spec is not None: + nullable_kind.spec = '?' + self.spec + nullable_kind.is_nullable = True + nullable_kind.parent_kind = self.parent_kind + nullable_kind.module = self.module + + return nullable_kind + + def MakeUnnullableKind(self): + assert self.is_nullable + + if self == NULLABLE_STRING: + return STRING + if self == NULLABLE_HANDLE: + return HANDLE + if self == NULLABLE_DCPIPE: + return DCPIPE + if self == NULLABLE_DPPIPE: + return DPPIPE + if self == NULLABLE_MSGPIPE: + return MSGPIPE + if self == NULLABLE_SHAREDBUFFER: + return SHAREDBUFFER + if self == NULLABLE_PLATFORMHANDLE: + return PLATFORMHANDLE + + unnullable_kind = type(self)() + unnullable_kind.shared_definition = self.shared_definition + if self.spec is not None: + assert self.spec[0] == '?' + unnullable_kind.spec = self.spec[1:] + unnullable_kind.is_nullable = False + unnullable_kind.parent_kind = self.parent_kind + unnullable_kind.module = self.module + + return unnullable_kind + + def __eq__(self, rhs): + return (isinstance(rhs, ReferenceKind) and super().__eq__(rhs)) + + def __hash__(self): # pylint: disable=useless-super-delegation + return super().__hash__() + + +# Initialize the set of primitive types. These can be accessed by clients. +BOOL = ValueKind('b') +INT8 = ValueKind('i8') +INT16 = ValueKind('i16') +INT32 = ValueKind('i32') +INT64 = ValueKind('i64') +UINT8 = ValueKind('u8') +UINT16 = ValueKind('u16') +UINT32 = ValueKind('u32') +UINT64 = ValueKind('u64') +FLOAT = ValueKind('f') +DOUBLE = ValueKind('d') +NULLABLE_BOOL = ValueKind('?b', True) +NULLABLE_INT8 = ValueKind('?i8', True) +NULLABLE_INT16 = ValueKind('?i16', True) +NULLABLE_INT32 = ValueKind('?i32', True) +NULLABLE_INT64 = ValueKind('?i64', True) +NULLABLE_UINT8 = ValueKind('?u8', True) +NULLABLE_UINT16 = ValueKind('?u16', True) +NULLABLE_UINT32 = ValueKind('?u32', True) +NULLABLE_UINT64 = ValueKind('?u64', True) +NULLABLE_FLOAT = ValueKind('?f', True) +NULLABLE_DOUBLE = ValueKind('?d', True) +STRING = ReferenceKind('s') +HANDLE = ReferenceKind('h') +DCPIPE = ReferenceKind('h:d:c') +DPPIPE = ReferenceKind('h:d:p') +MSGPIPE = ReferenceKind('h:m') +SHAREDBUFFER = ReferenceKind('h:s') +PLATFORMHANDLE = ReferenceKind('h:p') +NULLABLE_STRING = ReferenceKind('?s', True) +NULLABLE_HANDLE = ReferenceKind('?h', True) +NULLABLE_DCPIPE = ReferenceKind('?h:d:c', True) +NULLABLE_DPPIPE = ReferenceKind('?h:d:p', True) +NULLABLE_MSGPIPE = ReferenceKind('?h:m', True) +NULLABLE_SHAREDBUFFER = ReferenceKind('?h:s', True) +NULLABLE_PLATFORMHANDLE = ReferenceKind('?h:p', True) + +# Collection of all Primitive types +PRIMITIVES = ( + BOOL, + INT8, + INT16, + INT32, + INT64, + UINT8, + UINT16, + UINT32, + UINT64, + FLOAT, + DOUBLE, + NULLABLE_BOOL, + NULLABLE_INT8, + NULLABLE_INT16, + NULLABLE_INT32, + NULLABLE_INT64, + NULLABLE_UINT8, + NULLABLE_UINT16, + NULLABLE_UINT32, + NULLABLE_UINT64, + NULLABLE_FLOAT, + NULLABLE_DOUBLE, + STRING, + HANDLE, + DCPIPE, + DPPIPE, + MSGPIPE, + SHAREDBUFFER, + PLATFORMHANDLE, + NULLABLE_STRING, + NULLABLE_HANDLE, + NULLABLE_DCPIPE, + NULLABLE_DPPIPE, + NULLABLE_MSGPIPE, + NULLABLE_SHAREDBUFFER, + NULLABLE_PLATFORMHANDLE, +) + +ATTRIBUTE_MIN_VERSION = 'MinVersion' +ATTRIBUTE_DEFAULT = 'Default' +ATTRIBUTE_EXTENSIBLE = 'Extensible' +ATTRIBUTE_NO_INTERRUPT = 'NoInterrupt' +ATTRIBUTE_STABLE = 'Stable' +ATTRIBUTE_SUPPORTS_URGENT = 'SupportsUrgent' +ATTRIBUTE_SYNC = 'Sync' +ATTRIBUTE_UNLIMITED_SIZE = 'UnlimitedSize' +ATTRIBUTE_UUID = 'Uuid' +ATTRIBUTE_SERVICE_SANDBOX = 'ServiceSandbox' +ATTRIBUTE_REQUIRE_CONTEXT = 'RequireContext' +ATTRIBUTE_ALLOWED_CONTEXT = 'AllowedContext' +ATTRIBUTE_RUNTIME_FEATURE = 'RuntimeFeature' + + +class NamedValue: + def __init__(self, module, parent_kind, mojom_name): + self.module = module + self.parent_kind = parent_kind + self.mojom_name = mojom_name + + def GetSpec(self): + return (self.module.GetNamespacePrefix() + + (self.parent_kind and + (self.parent_kind.mojom_name + '.') or "") + self.mojom_name) + + def __eq__(self, rhs): + return (isinstance(rhs, NamedValue) + and (self.parent_kind, self.mojom_name) == (rhs.parent_kind, + rhs.mojom_name)) + + def __hash__(self): + return hash((self.parent_kind, self.mojom_name)) + + +class BuiltinValue: + def __init__(self, value): + self.value = value + + def __eq__(self, rhs): + return isinstance(rhs, BuiltinValue) and self.value == rhs.value + + +class ConstantValue(NamedValue): + def __init__(self, module, parent_kind, constant): + NamedValue.__init__(self, module, parent_kind, constant.mojom_name) + self.constant = constant + + @property + def name(self): + return self.constant.name + + +class EnumValue(NamedValue): + def __init__(self, module, enum, field): + NamedValue.__init__(self, module, enum.parent_kind, field.mojom_name) + self.field = field + self.enum = enum + + def GetSpec(self): + return (self.module.GetNamespacePrefix() + + (self.parent_kind and (self.parent_kind.mojom_name + '.') or "") + + self.enum.mojom_name + '.' + self.mojom_name) + + @property + def name(self): + return self.field.name + + +class Constant: + def __init__(self, mojom_name=None, kind=None, value=None, parent_kind=None): + self.mojom_name = mojom_name + self.name = None + self.kind = kind + self.value = value + self.parent_kind = parent_kind + + def Stylize(self, stylizer): + self.name = stylizer.StylizeConstant(self.mojom_name) + + def __eq__(self, rhs): + return (isinstance(rhs, Constant) + and (self.mojom_name, self.kind, self.value, + self.parent_kind) == (rhs.mojom_name, rhs.kind, rhs.value, + rhs.parent_kind)) + + +class Field: + def __init__(self, + mojom_name=None, + kind=None, + ordinal=None, + default=None, + attributes=None): + if self.__class__.__name__ == 'Field': + raise Exception() + self.mojom_name = mojom_name + self.name = None + self.kind = kind + self.ordinal = ordinal + self.default = default + self.attributes = attributes + + def Repr(self, as_ref=True): + # pylint: disable=unused-argument + # Fields are only referenced by objects which define them and thus + # they are always displayed as non-references. + return GenericRepr(self, {'mojom_name': False, 'kind': True}) + + def Stylize(self, stylizer): + self.name = stylizer.StylizeField(self.mojom_name) + + @property + def min_version(self): + return self.attributes.get(ATTRIBUTE_MIN_VERSION) \ + if self.attributes else None + + def __eq__(self, rhs): + return (isinstance(rhs, Field) + and (self.mojom_name, self.kind, self.ordinal, self.default, + self.attributes) == (rhs.mojom_name, rhs.kind, rhs.ordinal, + rhs.default, rhs.attributes)) + + def __hash__(self): + return hash((self.mojom_name, self.kind, self.ordinal, self.default)) + + +class StructField(Field): + def __hash__(self): + return super(Field, self).__hash__() + + +class UnionField(Field): + def __init__(self, + mojom_name=None, + kind=None, + ordinal=None, + default=None, + attributes=None): + Field.__init__(self, mojom_name, kind, ordinal, default, attributes) + + @property + def is_default(self): + return self.attributes.get(ATTRIBUTE_DEFAULT, False) \ + if self.attributes else False + + +def _IsFieldBackwardCompatible(new_field, old_field, checker): + if (new_field.min_version or 0) != (old_field.min_version or 0): + return False + + return checker.IsBackwardCompatible(new_field.kind, old_field.kind) + + +class Feature(ReferenceKind): + """A runtime enabled feature defined from mojom. + + Attributes: + mojom_name: {str} The name of the feature type as defined in mojom. + name: {str} The stylized name. (Note: not the "name" used by FeatureList.) + constants: {List[Constant]} The constants defined in the feature scope. + attributes: {dict} Additional information about the feature. + """ + + Kind.AddSharedProperty('mojom_name') + Kind.AddSharedProperty('name') + Kind.AddSharedProperty('constants') + Kind.AddSharedProperty('attributes') + + def __init__(self, mojom_name=None, module=None, attributes=None): + if mojom_name is not None: + spec = 'x:' + mojom_name + else: + spec = None + ReferenceKind.__init__(self, spec, False, module) + self.mojom_name = mojom_name + self.name = None + self.constants = [] + self.attributes = attributes + + def Stylize(self, stylizer): + self.name = stylizer.StylizeFeature(self.mojom_name) + for constant in self.constants: + constant.Stylize(stylizer) + + +class Struct(ReferenceKind): + """A struct with typed fields. + + Attributes: + mojom_name: {str} The name of the struct type as defined in mojom. + name: {str} The stylized name. + native_only: {bool} Does the struct have a body (i.e. any fields) or is it + purely a native struct. + custom_serializer: {bool} Should we generate a serializer for the struct or + will one be provided by non-generated code. + fields: {List[StructField]} The members of the struct. + enums: {List[Enum]} The enums defined in the struct scope. + constants: {List[Constant]} The constants defined in the struct scope. + attributes: {dict} Additional information about the struct, such as + if it's a native struct. + """ + + Kind.AddSharedProperty('mojom_name') + Kind.AddSharedProperty('name') + Kind.AddSharedProperty('native_only') + Kind.AddSharedProperty('custom_serializer') + Kind.AddSharedProperty('fields') + Kind.AddSharedProperty('enums') + Kind.AddSharedProperty('constants') + Kind.AddSharedProperty('attributes') + + def __init__(self, mojom_name=None, module=None, attributes=None): + if mojom_name is not None: + spec = 'x:' + mojom_name + else: + spec = None + ReferenceKind.__init__(self, spec, False, module) + self.mojom_name = mojom_name + self.name = None + self.native_only = False + self.custom_serializer = False + self.fields = [] + self.enums = [] + self.constants = [] + self.attributes = attributes + + def Repr(self, as_ref=True): + if as_ref: + return '<%s mojom_name=%r module=%s>' % (self.__class__.__name__, + self.mojom_name, + Repr(self.module, as_ref=True)) + return GenericRepr(self, { + 'mojom_name': False, + 'fields': False, + 'module': True + }) + + def AddField(self, + mojom_name, + kind, + ordinal=None, + default=None, + attributes=None): + field = StructField(mojom_name, kind, ordinal, default, attributes) + self.fields.append(field) + return field + + def Stylize(self, stylizer): + self.name = stylizer.StylizeStruct(self.mojom_name) + for field in self.fields: + field.Stylize(stylizer) + for enum in self.enums: + enum.Stylize(stylizer) + for constant in self.constants: + constant.Stylize(stylizer) + + def IsBackwardCompatible(self, rhs, checker): + """This struct is backward-compatible with rhs (older_struct) if and only if + all of the following conditions hold: + - Any newly added field is tagged with a [MinVersion] attribute specifying + a version number greater than all previously used [MinVersion] + attributes within the struct. + - All fields present in rhs remain present in the new struct, + with the same ordinal position, same optional or non-optional status, + same (or backward-compatible) type and where applicable, the same + [MinVersion] attribute value. + - All [MinVersion] attributes must be non-decreasing in ordinal order. + - All reference-typed (string, array, map, struct, or union) fields tagged + with a [MinVersion] greater than zero must be optional. + """ + + def buildOrdinalFieldMap(struct): + fields_by_ordinal = {} + for field in struct.fields: + if field.ordinal in fields_by_ordinal: + raise Exception('Multiple fields with ordinal %s in struct %s.' % + (field.ordinal, struct.mojom_name)) + fields_by_ordinal[field.ordinal] = field + return fields_by_ordinal + + new_fields = buildOrdinalFieldMap(self) + old_fields = buildOrdinalFieldMap(rhs) + if len(new_fields) < len(old_fields): + # At least one field was removed, which is not OK. + return False + + # If there are N fields, existing ordinal values must exactly cover the + # range from 0 to N-1. + num_old_ordinals = len(old_fields) + max_old_min_version = 0 + for ordinal in range(num_old_ordinals): + new_field = new_fields[ordinal] + old_field = old_fields[ordinal] + if (old_field.min_version or 0) > max_old_min_version: + max_old_min_version = old_field.min_version + if not _IsFieldBackwardCompatible(new_field, old_field, checker): + # Type or min-version mismatch between old and new versions of the same + # ordinal field. + return False + + # At this point we know all old fields are intact in the new struct + # definition. Now verify that all new fields have a high enough min version + # and are appropriately optional where required. + num_new_ordinals = len(new_fields) + last_min_version = max_old_min_version + for ordinal in range(num_old_ordinals, num_new_ordinals): + new_field = new_fields[ordinal] + min_version = new_field.min_version or 0 + if min_version <= max_old_min_version: + # A new field is being added to an existing version, which is not OK. + return False + if min_version < last_min_version: + # The [MinVersion] of a field cannot be lower than the [MinVersion] of + # a field with lower ordinal value. + return False + if IsReferenceKind(new_field.kind) and not IsNullableKind(new_field.kind): + # New fields whose type can be nullable MUST be nullable. + return False + + return True + + @property + def stable(self): + return self.attributes.get(ATTRIBUTE_STABLE, False) \ + if self.attributes else False + + @property + def qualified_name(self): + if self.parent_kind: + prefix = self.parent_kind.qualified_name + '.' + else: + prefix = self.module.GetNamespacePrefix() + return '%s%s' % (prefix, self.mojom_name) + + def _tuple(self): + return (self.mojom_name, self.native_only, self.fields, self.constants, + self.attributes) + + def __eq__(self, rhs): + return isinstance(rhs, Struct) and self._tuple() == rhs._tuple() + + def __lt__(self, rhs): + if not isinstance(self, type(rhs)): + return str(type(self)) < str(type(rhs)) + + return self._tuple() < rhs._tuple() + + def __hash__(self): + return id(self) + + +class Union(ReferenceKind): + """A union of several kinds. + + Attributes: + mojom_name: {str} The name of the union type as defined in mojom. + name: {str} The stylized name. + fields: {List[UnionField]} The members of the union. + attributes: {dict} Additional information about the union, such as + which Java class name to use to represent it in the generated + bindings. + """ + Kind.AddSharedProperty('mojom_name') + Kind.AddSharedProperty('name') + Kind.AddSharedProperty('fields') + Kind.AddSharedProperty('attributes') + Kind.AddSharedProperty('default_field') + + def __init__(self, mojom_name=None, module=None, attributes=None): + if mojom_name is not None: + spec = 'x:' + mojom_name + else: + spec = None + ReferenceKind.__init__(self, spec, False, module) + self.mojom_name = mojom_name + self.name = None + self.fields = [] + self.attributes = attributes + self.default_field = None + + def Repr(self, as_ref=True): + if as_ref: + return '<%s spec=%r is_nullable=%r fields=%s>' % ( + self.__class__.__name__, self.spec, self.is_nullable, Repr( + self.fields)) + return GenericRepr(self, {'fields': True, 'is_nullable': False}) + + def AddField(self, mojom_name, kind, ordinal=None, attributes=None): + field = UnionField(mojom_name, kind, ordinal, None, attributes) + self.fields.append(field) + return field + + def Stylize(self, stylizer): + self.name = stylizer.StylizeUnion(self.mojom_name) + for field in self.fields: + field.Stylize(stylizer) + + def IsBackwardCompatible(self, rhs, checker): + """This union is backward-compatible with rhs (older_union) if and only if + all of the following conditions hold: + - Any newly added field is tagged with a [MinVersion] attribute specifying + a version number greater than all previously used [MinVersion] + attributes within the union. + - All fields present in rhs remain present in the new union, + with the same ordinal value, same optional or non-optional status, + same (or backward-compatible) type, and where applicable, the same + [MinVersion] attribute value. + """ + + def buildOrdinalFieldMap(union): + fields_by_ordinal = {} + for field in union.fields: + if field.ordinal in fields_by_ordinal: + raise Exception('Multiple fields with ordinal %s in union %s.' % + (field.ordinal, union.mojom_name)) + fields_by_ordinal[field.ordinal] = field + return fields_by_ordinal + + new_fields = buildOrdinalFieldMap(self) + old_fields = buildOrdinalFieldMap(rhs) + if len(new_fields) < len(old_fields): + # At least one field was removed, which is not OK. + return False + + max_old_min_version = 0 + for ordinal, old_field in old_fields.items(): + new_field = new_fields.get(ordinal) + if not new_field: + # A field was removed, which is not OK. + return False + if not _IsFieldBackwardCompatible(new_field, old_field, checker): + # An field changed its type or MinVersion, which is not OK. + return False + old_min_version = old_field.min_version or 0 + if old_min_version > max_old_min_version: + max_old_min_version = old_min_version + + new_ordinals = set(new_fields.keys()) - set(old_fields.keys()) + for ordinal in new_ordinals: + if (new_fields[ordinal].min_version or 0) <= max_old_min_version: + # New fields must use a MinVersion greater than any old fields. + return False + + return True + + @property + def extensible(self): + return self.attributes.get(ATTRIBUTE_EXTENSIBLE, False) \ + if self.attributes else False + + @property + def stable(self): + return self.attributes.get(ATTRIBUTE_STABLE, False) \ + if self.attributes else False + + @property + def qualified_name(self): + if self.parent_kind: + prefix = self.parent_kind.qualified_name + '.' + else: + prefix = self.module.GetNamespacePrefix() + return '%s%s' % (prefix, self.mojom_name) + + def _tuple(self): + return (self.mojom_name, self.fields, self.attributes) + + def __eq__(self, rhs): + return isinstance(rhs, Union) and self._tuple() == rhs._tuple() + + def __lt__(self, rhs): + if not isinstance(self, type(rhs)): + return str(type(self)) < str(type(rhs)) + + return self._tuple() < rhs._tuple() + + def __hash__(self): + return id(self) + + +class Array(ReferenceKind): + """An array. + + Attributes: + kind: {Kind} The type of the elements. May be None. + length: The number of elements. None if unknown. + """ + + Kind.AddSharedProperty('kind') + Kind.AddSharedProperty('length') + + def __init__(self, kind=None, length=None): + if kind is not None: + if length is not None: + spec = 'a%d:%s' % (length, kind.spec) + else: + spec = 'a:%s' % kind.spec + + ReferenceKind.__init__(self, spec) + else: + ReferenceKind.__init__(self) + self.kind = kind + self.length = length + + def Repr(self, as_ref=True): + if as_ref: + return '<%s spec=%r is_nullable=%r kind=%s length=%r>' % ( + self.__class__.__name__, self.spec, self.is_nullable, Repr( + self.kind), self.length) + return GenericRepr(self, { + 'kind': True, + 'length': False, + 'is_nullable': False + }) + + def __eq__(self, rhs): + return (isinstance(rhs, Array) + and (self.kind, self.length) == (rhs.kind, rhs.length)) + + def __hash__(self): + return id(self) + + def IsBackwardCompatible(self, rhs, checker): + return (isinstance(rhs, Array) and self.length == rhs.length + and checker.IsBackwardCompatible(self.kind, rhs.kind)) + + +class Map(ReferenceKind): + """A map. + + Attributes: + key_kind: {Kind} The type of the keys. May be None. + value_kind: {Kind} The type of the elements. May be None. + """ + Kind.AddSharedProperty('key_kind') + Kind.AddSharedProperty('value_kind') + + def __init__(self, key_kind=None, value_kind=None): + if (key_kind is not None and value_kind is not None): + ReferenceKind.__init__( + self, 'm[' + key_kind.spec + '][' + value_kind.spec + ']') + if IsNullableKind(key_kind): + raise Exception("Nullable kinds cannot be keys in maps.") + if IsAnyHandleKind(key_kind): + raise Exception("Handles cannot be keys in maps.") + if IsAnyInterfaceKind(key_kind): + raise Exception("Interfaces cannot be keys in maps.") + if IsArrayKind(key_kind): + raise Exception("Arrays cannot be keys in maps.") + else: + ReferenceKind.__init__(self) + + self.key_kind = key_kind + self.value_kind = value_kind + + def Repr(self, as_ref=True): + if as_ref: + return '<%s spec=%r is_nullable=%r key_kind=%s value_kind=%s>' % ( + self.__class__.__name__, self.spec, self.is_nullable, + Repr(self.key_kind), Repr(self.value_kind)) + return GenericRepr(self, {'key_kind': True, 'value_kind': True}) + + def __eq__(self, rhs): + return (isinstance(rhs, Map) and + (self.key_kind, self.value_kind) == (rhs.key_kind, rhs.value_kind)) + + def __hash__(self): + return id(self) + + def IsBackwardCompatible(self, rhs, checker): + return (isinstance(rhs, Map) + and checker.IsBackwardCompatible(self.key_kind, rhs.key_kind) + and checker.IsBackwardCompatible(self.value_kind, rhs.value_kind)) + + +class PendingRemote(ReferenceKind): + Kind.AddSharedProperty('kind') + + def __init__(self, kind=None): + if kind is not None: + if not isinstance(kind, Interface): + raise Exception( + 'pending_remote requires T to be an interface type. Got %r' % + kind.spec) + ReferenceKind.__init__(self, 'rmt:' + kind.spec) + else: + ReferenceKind.__init__(self) + self.kind = kind + + def __eq__(self, rhs): + return isinstance(rhs, PendingRemote) and self.kind == rhs.kind + + def __hash__(self): + return id(self) + + def IsBackwardCompatible(self, rhs, checker): + return (isinstance(rhs, PendingRemote) + and checker.IsBackwardCompatible(self.kind, rhs.kind)) + + +class PendingReceiver(ReferenceKind): + Kind.AddSharedProperty('kind') + + def __init__(self, kind=None): + if kind is not None: + if not isinstance(kind, Interface): + raise Exception( + 'pending_receiver requires T to be an interface type. Got %r' % + kind.spec) + ReferenceKind.__init__(self, 'rcv:' + kind.spec) + else: + ReferenceKind.__init__(self) + self.kind = kind + + def __eq__(self, rhs): + return isinstance(rhs, PendingReceiver) and self.kind == rhs.kind + + def __hash__(self): + return id(self) + + def IsBackwardCompatible(self, rhs, checker): + return isinstance(rhs, PendingReceiver) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + + +class PendingAssociatedRemote(ReferenceKind): + Kind.AddSharedProperty('kind') + + def __init__(self, kind=None): + if kind is not None: + if not isinstance(kind, Interface): + raise Exception( + 'pending_associated_remote requires T to be an interface ' + + 'type. Got %r' % kind.spec) + ReferenceKind.__init__(self, 'rma:' + kind.spec) + else: + ReferenceKind.__init__(self) + self.kind = kind + + def __eq__(self, rhs): + return isinstance(rhs, PendingAssociatedRemote) and self.kind == rhs.kind + + def __hash__(self): + return id(self) + + def IsBackwardCompatible(self, rhs, checker): + return isinstance(rhs, + PendingAssociatedRemote) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + + +class PendingAssociatedReceiver(ReferenceKind): + Kind.AddSharedProperty('kind') + + def __init__(self, kind=None): + if kind is not None: + if not isinstance(kind, Interface): + raise Exception( + 'pending_associated_receiver requires T to be an interface' + + 'type. Got %r' % kind.spec) + ReferenceKind.__init__(self, 'rca:' + kind.spec) + else: + ReferenceKind.__init__(self) + self.kind = kind + + def __eq__(self, rhs): + return isinstance(rhs, PendingAssociatedReceiver) and self.kind == rhs.kind + + def __hash__(self): + return id(self) + + def IsBackwardCompatible(self, rhs, checker): + return isinstance( + rhs, PendingAssociatedReceiver) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + + +class InterfaceRequest(ReferenceKind): + Kind.AddSharedProperty('kind') + + def __init__(self, kind=None): + if kind is not None: + if not isinstance(kind, Interface): + raise Exception( + "Interface request requires %r to be an interface." % kind.spec) + ReferenceKind.__init__(self, 'r:' + kind.spec) + else: + ReferenceKind.__init__(self) + self.kind = kind + + def __eq__(self, rhs): + return isinstance(rhs, InterfaceRequest) and self.kind == rhs.kind + + def __hash__(self): + return id(self) + + def IsBackwardCompatible(self, rhs, checker): + return isinstance(rhs, InterfaceRequest) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + + +class AssociatedInterfaceRequest(ReferenceKind): + Kind.AddSharedProperty('kind') + + def __init__(self, kind=None): + if kind is not None: + if not isinstance(kind, InterfaceRequest): + raise Exception( + "Associated interface request requires %r to be an interface " + "request." % kind.spec) + assert not kind.is_nullable + ReferenceKind.__init__(self, 'asso:' + kind.spec) + else: + ReferenceKind.__init__(self) + self.kind = kind.kind if kind is not None else None + + def __eq__(self, rhs): + return isinstance(rhs, AssociatedInterfaceRequest) and self.kind == rhs.kind + + def __hash__(self): + return id(self) + + def IsBackwardCompatible(self, rhs, checker): + return isinstance( + rhs, AssociatedInterfaceRequest) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + + +class Parameter: + def __init__(self, + mojom_name=None, + kind=None, + ordinal=None, + default=None, + attributes=None): + self.mojom_name = mojom_name + self.name = None + self.ordinal = ordinal + self.kind = kind + self.default = default + self.attributes = attributes + + def Repr(self, as_ref=True): + # pylint: disable=unused-argument + return '<%s mojom_name=%r kind=%s>' % ( + self.__class__.__name__, self.mojom_name, self.kind.Repr(as_ref=True)) + + def Stylize(self, stylizer): + self.name = stylizer.StylizeParameter(self.mojom_name) + + @property + def min_version(self): + return self.attributes.get(ATTRIBUTE_MIN_VERSION) \ + if self.attributes else None + + def __eq__(self, rhs): + return (isinstance(rhs, Parameter) + and (self.mojom_name, self.ordinal, self.kind, self.default, + self.attributes) == (rhs.mojom_name, rhs.ordinal, rhs.kind, + rhs.default, rhs.attributes)) + + +class Method: + def __init__(self, interface, mojom_name, ordinal=None, attributes=None): + self.interface = interface + self.mojom_name = mojom_name + self.name = None + self.explicit_ordinal = ordinal + self.ordinal = ordinal + self.parameters = [] + self.param_struct = None + self.response_parameters = None + self.response_param_struct = None + self.attributes = attributes + + def Repr(self, as_ref=True): + if as_ref: + return '<%s mojom_name=%r>' % (self.__class__.__name__, self.mojom_name) + return GenericRepr(self, { + 'mojom_name': False, + 'parameters': True, + 'response_parameters': True + }) + + def AddParameter(self, + mojom_name, + kind, + ordinal=None, + default=None, + attributes=None): + parameter = Parameter(mojom_name, kind, ordinal, default, attributes) + self.parameters.append(parameter) + return parameter + + def AddResponseParameter(self, + mojom_name, + kind, + ordinal=None, + default=None, + attributes=None): + if self.response_parameters == None: + self.response_parameters = [] + parameter = Parameter(mojom_name, kind, ordinal, default, attributes) + self.response_parameters.append(parameter) + return parameter + + def Stylize(self, stylizer): + self.name = stylizer.StylizeMethod(self.mojom_name) + for param in self.parameters: + param.Stylize(stylizer) + if self.response_parameters is not None: + for param in self.response_parameters: + param.Stylize(stylizer) + + if self.param_struct: + self.param_struct.Stylize(stylizer) + if self.response_param_struct: + self.response_param_struct.Stylize(stylizer) + + @property + def min_version(self): + return self.attributes.get(ATTRIBUTE_MIN_VERSION) \ + if self.attributes else None + + @property + def sync(self): + return self.attributes.get(ATTRIBUTE_SYNC) \ + if self.attributes else None + + @property + def allow_interrupt(self): + return not self.attributes.get(ATTRIBUTE_NO_INTERRUPT) \ + if self.attributes else True + + @property + def unlimited_message_size(self): + return self.attributes.get(ATTRIBUTE_UNLIMITED_SIZE) \ + if self.attributes else False + + @property + def allowed_context(self): + return self.attributes.get(ATTRIBUTE_ALLOWED_CONTEXT) \ + if self.attributes else None + + @property + def supports_urgent(self): + return self.attributes.get(ATTRIBUTE_SUPPORTS_URGENT) \ + if self.attributes else None + + @property + def runtime_feature(self): + if not self.attributes: + return None + runtime_feature = self.attributes.get(ATTRIBUTE_RUNTIME_FEATURE, None) + if runtime_feature is None: + return None + if not isinstance(runtime_feature, Feature): + raise Exception("RuntimeFeature attribute on %s must be a feature." % + self.name) + return runtime_feature + + def _tuple(self): + return (self.mojom_name, self.ordinal, self.parameters, + self.response_parameters, self.attributes) + + def __eq__(self, rhs): + return isinstance(rhs, Method) and self._tuple() == rhs._tuple() + + def __lt__(self, rhs): + if not isinstance(self, type(rhs)): + return str(type(self)) < str(type(rhs)) + + return self._tuple() < rhs._tuple() + + +class Interface(ReferenceKind): + Kind.AddSharedProperty('mojom_name') + Kind.AddSharedProperty('name') + Kind.AddSharedProperty('methods') + Kind.AddSharedProperty('enums') + Kind.AddSharedProperty('constants') + Kind.AddSharedProperty('attributes') + + def __init__(self, mojom_name=None, module=None, attributes=None): + if mojom_name is not None: + spec = 'x:' + mojom_name + else: + spec = None + ReferenceKind.__init__(self, spec, False, module) + self.mojom_name = mojom_name + self.name = None + self.methods = [] + self.enums = [] + self.constants = [] + self.attributes = attributes + + def Repr(self, as_ref=True): + if as_ref: + return '<%s mojom_name=%r>' % (self.__class__.__name__, self.mojom_name) + return GenericRepr(self, { + 'mojom_name': False, + 'attributes': False, + 'methods': False + }) + + def AddMethod(self, mojom_name, ordinal=None, attributes=None): + method = Method(self, mojom_name, ordinal, attributes) + self.methods.append(method) + return method + + def Stylize(self, stylizer): + self.name = stylizer.StylizeInterface(self.mojom_name) + for method in self.methods: + method.Stylize(stylizer) + for enum in self.enums: + enum.Stylize(stylizer) + for constant in self.constants: + constant.Stylize(stylizer) + + def IsBackwardCompatible(self, rhs, checker): + """This interface is backward-compatible with rhs (older_interface) if and + only if all of the following conditions hold: + - All defined methods in rhs (when identified by ordinal) have + backward-compatible definitions in this interface. For each method this + means: + - The parameter list is backward-compatible, according to backward- + compatibility rules for structs, where each parameter is essentially + a struct field. + - If the old method definition does not specify a reply message, the + new method definition must not specify a reply message. + - If the old method definition specifies a reply message, the new + method definition must also specify a reply message with a parameter + list that is backward-compatible according to backward-compatibility + rules for structs. + - All newly introduced methods in this interface have a [MinVersion] + attribute specifying a version greater than any method in + rhs. + """ + + def buildOrdinalMethodMap(interface): + methods_by_ordinal = {} + for method in interface.methods: + if method.ordinal in methods_by_ordinal: + raise Exception('Multiple methods with ordinal %s in interface %s.' % + (method.ordinal, interface.mojom_name)) + methods_by_ordinal[method.ordinal] = method + return methods_by_ordinal + + new_methods = buildOrdinalMethodMap(self) + old_methods = buildOrdinalMethodMap(rhs) + max_old_min_version = 0 + for ordinal, old_method in old_methods.items(): + new_method = new_methods.get(ordinal) + if not new_method: + # A method was removed, which is not OK. + return False + + if not checker.IsBackwardCompatible(new_method.param_struct, + old_method.param_struct): + # The parameter list is not backward-compatible, which is not OK. + return False + + if old_method.response_param_struct is None: + if new_method.response_param_struct is not None: + # A reply was added to a message which didn't have one before, and + # this is not OK. + return False + else: + if new_method.response_param_struct is None: + # A reply was removed from a message, which is not OK. + return False + if not checker.IsBackwardCompatible(new_method.response_param_struct, + old_method.response_param_struct): + # The new message's reply is not backward-compatible with the old + # message's reply, which is not OK. + return False + + if (old_method.min_version or 0) > max_old_min_version: + max_old_min_version = old_method.min_version + + # All the old methods are compatible with their new counterparts. Now verify + # that newly added methods are properly versioned. + new_ordinals = set(new_methods.keys()) - set(old_methods.keys()) + for ordinal in new_ordinals: + new_method = new_methods[ordinal] + if (new_method.min_version or 0) <= max_old_min_version: + # A method was added to an existing version, which is not OK. + return False + + return True + + @property + def service_sandbox(self): + if not self.attributes: + return None + service_sandbox = self.attributes.get(ATTRIBUTE_SERVICE_SANDBOX, None) + if service_sandbox is None: + return None + # Constants are only allowed to refer to an enum here, so replace. + if isinstance(service_sandbox, Constant): + service_sandbox = service_sandbox.value + if not isinstance(service_sandbox, EnumValue): + raise Exception("ServiceSandbox attribute on %s must be an enum value." % + self.module.name) + return service_sandbox + + @property + def runtime_feature(self): + if not self.attributes: + return None + runtime_feature = self.attributes.get(ATTRIBUTE_RUNTIME_FEATURE, None) + if runtime_feature is None: + return None + if not isinstance(runtime_feature, Feature): + raise Exception("RuntimeFeature attribute on %s must be a feature." % + self.name) + return runtime_feature + + @property + def require_context(self): + if not self.attributes: + return None + return self.attributes.get(ATTRIBUTE_REQUIRE_CONTEXT, None) + + @property + def stable(self): + return self.attributes.get(ATTRIBUTE_STABLE, False) \ + if self.attributes else False + + @property + def qualified_name(self): + if self.parent_kind: + prefix = self.parent_kind.qualified_name + '.' + else: + prefix = self.module.GetNamespacePrefix() + return '%s%s' % (prefix, self.mojom_name) + + def _tuple(self): + return (self.mojom_name, self.methods, self.enums, self.constants, + self.attributes) + + def __eq__(self, rhs): + return isinstance(rhs, Interface) and self._tuple() == rhs._tuple() + + def __lt__(self, rhs): + if not isinstance(self, type(rhs)): + return str(type(self)) < str(type(rhs)) + + return self._tuple() < rhs._tuple() + + @property + def uuid(self): + uuid_str = self.attributes.get(ATTRIBUTE_UUID) if self.attributes else None + if uuid_str is None: + return None + + try: + u = UUID(uuid_str) + except: + raise ValueError('Invalid format for Uuid attribute on interface {}. ' + 'Expected standard RFC 4122 string representation of ' + 'a UUID.'.format(self.mojom_name)) + return (int(u.hex[:16], 16), int(u.hex[16:], 16)) + + def __hash__(self): + return id(self) + + +class AssociatedInterface(ReferenceKind): + Kind.AddSharedProperty('kind') + + def __init__(self, kind=None): + if kind is not None: + if not isinstance(kind, Interface): + raise Exception( + "Associated interface requires %r to be an interface." % kind.spec) + assert not kind.is_nullable + ReferenceKind.__init__(self, 'asso:' + kind.spec) + else: + ReferenceKind.__init__(self) + self.kind = kind + + def __eq__(self, rhs): + return isinstance(rhs, AssociatedInterface) and self.kind == rhs.kind + + def __hash__(self): + return id(self) + + def IsBackwardCompatible(self, rhs, checker): + return isinstance(rhs, + AssociatedInterface) and checker.IsBackwardCompatible( + self.kind, rhs.kind) + + +class EnumField: + def __init__(self, + mojom_name=None, + value=None, + attributes=None, + numeric_value=None): + self.mojom_name = mojom_name + self.name = None + self.value = value + self.attributes = attributes + self.numeric_value = numeric_value + + def Stylize(self, stylizer): + self.name = stylizer.StylizeEnumField(self.mojom_name) + + @property + def default(self): + return self.attributes.get(ATTRIBUTE_DEFAULT, False) \ + if self.attributes else False + + @property + def min_version(self): + return self.attributes.get(ATTRIBUTE_MIN_VERSION) \ + if self.attributes else None + + def __eq__(self, rhs): + return (isinstance(rhs, EnumField) + and (self.mojom_name, self.value, self.attributes, + self.numeric_value) == (rhs.mojom_name, rhs.value, + rhs.attributes, rhs.numeric_value)) + + +class Enum(ValueKind): + Kind.AddSharedProperty('mojom_name') + Kind.AddSharedProperty('name') + Kind.AddSharedProperty('native_only') + Kind.AddSharedProperty('fields') + Kind.AddSharedProperty('attributes') + Kind.AddSharedProperty('min_value') + Kind.AddSharedProperty('max_value') + Kind.AddSharedProperty('default_field') + + def __init__(self, mojom_name=None, module=None, attributes=None): + if mojom_name is not None: + spec = 'x:' + mojom_name + else: + spec = None + ValueKind.__init__(self, spec, False, module) + self.mojom_name = mojom_name + self.name = None + self.native_only = False + self.fields = [] + self.attributes = attributes + self.min_value = None + self.max_value = None + self.default_field = None + + def Repr(self, as_ref=True): + if as_ref: + return '<%s mojom_name=%r>' % (self.__class__.__name__, self.mojom_name) + return GenericRepr(self, {'mojom_name': False, 'fields': False}) + + def Stylize(self, stylizer): + self.name = stylizer.StylizeEnum(self.mojom_name) + for field in self.fields: + field.Stylize(stylizer) + + @property + def extensible(self): + return self.attributes.get(ATTRIBUTE_EXTENSIBLE, False) \ + if self.attributes else False + + @property + def stable(self): + return self.attributes.get(ATTRIBUTE_STABLE, False) \ + if self.attributes else False + + @property + def qualified_name(self): + if self.parent_kind: + prefix = self.parent_kind.qualified_name + '.' + else: + prefix = self.module.GetNamespacePrefix() + return '%s%s' % (prefix, self.mojom_name) + + # pylint: disable=unused-argument + def IsBackwardCompatible(self, rhs, checker): + """This enum is backward-compatible with rhs (older_enum) if and only if one + of the following conditions holds: + - Neither enum is [Extensible] and both have the exact same set of valid + numeric values. Field names and aliases for the same numeric value do + not affect compatibility. + - rhs is [Extensible], and for every version defined by + rhs, this enum has the exact same set of valid numeric values. + """ + + def buildVersionFieldMap(enum): + fields_by_min_version = {} + for field in enum.fields: + if field.min_version not in fields_by_min_version: + fields_by_min_version[field.min_version] = set() + fields_by_min_version[field.min_version].add(field.numeric_value) + return fields_by_min_version + + old_fields = buildVersionFieldMap(rhs) + new_fields = buildVersionFieldMap(self) + + if new_fields.keys() != old_fields.keys() and not rhs.extensible: + raise Exception("Non-extensible enum cannot be modified") + + for min_version, valid_values in old_fields.items(): + if min_version not in new_fields: + raise Exception('New values added to an extensible enum ' + 'do not specify MinVersion: %s' % new_fields) + + if (new_fields[min_version] != valid_values): + if (len(new_fields[min_version]) < len(valid_values)): + raise Exception('Removing values for an existing MinVersion %s ' + 'is not allowed' % min_version) + + raise Exception( + 'New values don\'t match old values' + 'for an existing MinVersion %s,' + ' please specify MinVersion equal to "Next version" ' + 'in the enum description' + ' for the following values:\n%s' % + (min_version, new_fields[min_version].difference(valid_values))) + return True + + def _tuple(self): + return (self.mojom_name, self.native_only, self.fields, self.attributes, + self.min_value, self.max_value, self.default_field) + + def __eq__(self, rhs): + return isinstance(rhs, Enum) and self._tuple() == rhs._tuple() + + def __lt__(self, rhs): + if not isinstance(self, type(rhs)): + return str(type(self)) < str(type(rhs)) + + return self._tuple() < rhs._tuple() + + def __hash__(self): + return id(self) + + +class Module: + def __init__(self, path=None, mojom_namespace=None, attributes=None): + self.path = path + self.mojom_namespace = mojom_namespace + self.namespace = None + self.structs = [] + self.unions = [] + self.interfaces = [] + self.enums = [] + self.features = [] + self.constants = [] + self.kinds = OrderedDict() + self.attributes = attributes + self.imports = [] + self.imported_kinds = OrderedDict() + self.metadata = OrderedDict() + + def __repr__(self): + # Gives us a decent __repr__ for modules. + return self.Repr() + + def __eq__(self, rhs): + return (isinstance(rhs, Module) + and (self.path, self.attributes, self.mojom_namespace, self.imports, + self.constants, self.enums, self.structs, self.unions, + self.interfaces, self.features) + == (rhs.path, rhs.attributes, rhs.mojom_namespace, rhs.imports, + rhs.constants, rhs.enums, rhs.structs, rhs.unions, + rhs.interfaces, rhs.features)) + + def __hash__(self): + return id(self) + + def Repr(self, as_ref=True): + if as_ref: + return '<%s path=%r mojom_namespace=%r>' % ( + self.__class__.__name__, self.path, self.mojom_namespace) + return GenericRepr( + self, { + 'path': False, + 'mojom_namespace': False, + 'attributes': False, + 'structs': False, + 'interfaces': False, + 'unions': False, + 'features': False, + }) + + def GetNamespacePrefix(self): + return '%s.' % self.mojom_namespace if self.mojom_namespace else '' + + def AddInterface(self, mojom_name, attributes=None): + interface = Interface(mojom_name, self, attributes) + self.interfaces.append(interface) + return interface + + def AddStruct(self, mojom_name, attributes=None): + struct = Struct(mojom_name, self, attributes) + self.structs.append(struct) + return struct + + def AddUnion(self, mojom_name, attributes=None): + union = Union(mojom_name, self, attributes) + self.unions.append(union) + return union + + def AddFeature(self, mojom_name, attributes=None): + feature = Feature(mojom_name, self, attributes) + self.features.append(feature) + return feature + + def Stylize(self, stylizer): + self.namespace = stylizer.StylizeModule(self.mojom_namespace) + for struct in self.structs: + struct.Stylize(stylizer) + for union in self.unions: + union.Stylize(stylizer) + for interface in self.interfaces: + interface.Stylize(stylizer) + for enum in self.enums: + enum.Stylize(stylizer) + for constant in self.constants: + constant.Stylize(stylizer) + for feature in self.features: + feature.Stylize(stylizer) + + for imported_module in self.imports: + imported_module.Stylize(stylizer) + + def Dump(self, f): + pickle.dump(self, f) + + @classmethod + def Load(cls, f): + result = pickle.load(f) + assert isinstance(result, Module) + return result + + +def IsBoolKind(kind): + return kind.spec == BOOL.spec or kind.spec == NULLABLE_BOOL.spec + + +def IsFloatKind(kind): + return kind.spec == FLOAT.spec or kind.spec == NULLABLE_FLOAT.spec + + +def IsDoubleKind(kind): + return kind.spec == DOUBLE.spec or kind.spec == NULLABLE_DOUBLE.spec + + +def IsIntegralKind(kind): + return (kind.spec == BOOL.spec or kind.spec == INT8.spec + or kind.spec == INT16.spec or kind.spec == INT32.spec + or kind.spec == INT64.spec or kind.spec == UINT8.spec + or kind.spec == UINT16.spec or kind.spec == UINT32.spec + or kind.spec == UINT64.spec or kind.spec == NULLABLE_BOOL.spec + or kind.spec == NULLABLE_INT8.spec or kind.spec == NULLABLE_INT16.spec + or kind.spec == NULLABLE_INT32.spec + or kind.spec == NULLABLE_INT64.spec + or kind.spec == NULLABLE_UINT8.spec + or kind.spec == NULLABLE_UINT16.spec + or kind.spec == NULLABLE_UINT32.spec + or kind.spec == NULLABLE_UINT64.spec) + + +def IsStringKind(kind): + return kind.spec == STRING.spec or kind.spec == NULLABLE_STRING.spec + + +def IsGenericHandleKind(kind): + return kind.spec == HANDLE.spec or kind.spec == NULLABLE_HANDLE.spec + + +def IsDataPipeConsumerKind(kind): + return kind.spec == DCPIPE.spec or kind.spec == NULLABLE_DCPIPE.spec + + +def IsDataPipeProducerKind(kind): + return kind.spec == DPPIPE.spec or kind.spec == NULLABLE_DPPIPE.spec + + +def IsMessagePipeKind(kind): + return kind.spec == MSGPIPE.spec or kind.spec == NULLABLE_MSGPIPE.spec + + +def IsSharedBufferKind(kind): + return (kind.spec == SHAREDBUFFER.spec + or kind.spec == NULLABLE_SHAREDBUFFER.spec) + + +def IsPlatformHandleKind(kind): + return (kind.spec == PLATFORMHANDLE.spec + or kind.spec == NULLABLE_PLATFORMHANDLE.spec) + + +def IsStructKind(kind): + return isinstance(kind, Struct) + + +def IsUnionKind(kind): + return isinstance(kind, Union) + + +def IsArrayKind(kind): + return isinstance(kind, Array) + + +def IsFeatureKind(kind): + return isinstance(kind, Feature) + + +def IsInterfaceKind(kind): + return isinstance(kind, Interface) + + +def IsAssociatedInterfaceKind(kind): + return isinstance(kind, AssociatedInterface) + + +def IsInterfaceRequestKind(kind): + return isinstance(kind, InterfaceRequest) + + +def IsAssociatedInterfaceRequestKind(kind): + return isinstance(kind, AssociatedInterfaceRequest) + + +def IsPendingRemoteKind(kind): + return isinstance(kind, PendingRemote) + + +def IsPendingReceiverKind(kind): + return isinstance(kind, PendingReceiver) + + +def IsPendingAssociatedRemoteKind(kind): + return isinstance(kind, PendingAssociatedRemote) + + +def IsPendingAssociatedReceiverKind(kind): + return isinstance(kind, PendingAssociatedReceiver) + + +def IsEnumKind(kind): + return isinstance(kind, Enum) + + +def IsValueKind(kind): + return isinstance(kind, ValueKind) + + +def IsReferenceKind(kind): + return isinstance(kind, ReferenceKind) + + +def IsNullableKind(kind): + return kind.is_nullable + + +def IsMapKind(kind): + return isinstance(kind, Map) + + +def IsObjectKind(kind): + return IsPointerKind(kind) or IsUnionKind(kind) + + +def IsPointerKind(kind): + return (IsStructKind(kind) or IsArrayKind(kind) or IsStringKind(kind) + or IsMapKind(kind)) + + +# Please note that it doesn't include any interface kind. +def IsAnyHandleKind(kind): + return (IsGenericHandleKind(kind) or IsDataPipeConsumerKind(kind) + or IsDataPipeProducerKind(kind) or IsMessagePipeKind(kind) + or IsSharedBufferKind(kind) or IsPlatformHandleKind(kind)) + + +def IsAnyInterfaceKind(kind): + return (IsInterfaceKind(kind) or IsInterfaceRequestKind(kind) + or IsAssociatedKind(kind) or IsPendingRemoteKind(kind) + or IsPendingReceiverKind(kind)) + + +def IsAnyHandleOrInterfaceKind(kind): + return IsAnyHandleKind(kind) or IsAnyInterfaceKind(kind) + + +def IsAssociatedKind(kind): + return (IsAssociatedInterfaceKind(kind) + or IsAssociatedInterfaceRequestKind(kind) + or IsPendingAssociatedRemoteKind(kind) + or IsPendingAssociatedReceiverKind(kind)) + + +def HasCallbacks(interface): + for method in interface.methods: + if method.response_parameters != None: + return True + return False + + +# Finds out whether an interface passes associated interfaces and associated +# interface requests. +def PassesAssociatedKinds(interface): + visited_kinds = set() + for method in interface.methods: + if MethodPassesAssociatedKinds(method, visited_kinds): + return True + return False + + +def _AnyMethodParameterRecursive(method, predicate, visited_kinds=None): + def _HasProperty(kind): + if kind in visited_kinds: + # No need to examine the kind again. + return False + visited_kinds.add(kind) + if predicate(kind): + return True + if IsArrayKind(kind): + return _HasProperty(kind.kind) + if IsStructKind(kind) or IsUnionKind(kind): + for field in kind.fields: + if _HasProperty(field.kind): + return True + if IsMapKind(kind): + if _HasProperty(kind.key_kind) or _HasProperty(kind.value_kind): + return True + return False + + if visited_kinds is None: + visited_kinds = set() + + for param in method.parameters: + if _HasProperty(param.kind): + return True + if method.response_parameters != None: + for param in method.response_parameters: + if _HasProperty(param.kind): + return True + return False + + +# Finds out whether a method passes associated interfaces and associated +# interface requests. +def MethodPassesAssociatedKinds(method, visited_kinds=None): + return _AnyMethodParameterRecursive( + method, IsAssociatedKind, visited_kinds=visited_kinds) + + +# Determines whether a method passes interfaces. +def MethodPassesInterfaces(method): + return _AnyMethodParameterRecursive(method, IsInterfaceKind) + + +def GetSyncMethodOrdinals(interface): + return [method.ordinal for method in interface.methods if method.sync] + + +def HasUninterruptableMethods(interface): + for method in interface.methods: + if not method.allow_interrupt: + return True + return False + + +def ContainsHandlesOrInterfaces(kind): + """Check if the kind contains any handles. + + This check is recursive so it checks all struct fields, containers elements, + etc. + + Args: + struct: {Kind} The kind to check. + + Returns: + {bool}: True if the kind contains handles. + """ + # We remember the types we already checked to avoid infinite recursion when + # checking recursive (or mutually recursive) types: + checked = set() + + def Check(kind): + if kind.spec in checked: + return False + checked.add(kind.spec) + if IsStructKind(kind): + return any(Check(field.kind) for field in kind.fields) + if IsUnionKind(kind): + return any(Check(field.kind) for field in kind.fields) + if IsAnyHandleKind(kind): + return True + if IsAnyInterfaceKind(kind): + return True + if IsArrayKind(kind): + return Check(kind.kind) + if IsMapKind(kind): + return Check(kind.key_kind) or Check(kind.value_kind) + return False + + return Check(kind) + + +def ContainsNativeTypes(kind): + """Check if the kind contains any native type (struct or enum). + + This check is recursive so it checks all struct fields, scoped interface + enums, etc. + + Args: + struct: {Kind} The kind to check. + + Returns: + {bool}: True if the kind contains native types. + """ + # We remember the types we already checked to avoid infinite recursion when + # checking recursive (or mutually recursive) types: + checked = set() + + def Check(kind): + if kind.spec in checked: + return False + checked.add(kind.spec) + if IsEnumKind(kind): + return kind.native_only + if IsStructKind(kind): + if kind.native_only: + return True + if any(enum.native_only for enum in kind.enums): + return True + return any(Check(field.kind) for field in kind.fields) + if IsUnionKind(kind): + return any(Check(field.kind) for field in kind.fields) + if IsInterfaceKind(kind): + return any(enum.native_only for enum in kind.enums) + if IsArrayKind(kind): + return Check(kind.kind) + if IsMapKind(kind): + return Check(kind.key_kind) or Check(kind.value_kind) + return False + + return Check(kind) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/module_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/module_unittest.py new file mode 100644 index 00000000..2a4e852c --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/module_unittest.py @@ -0,0 +1,31 @@ +# Copyright 2014 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import sys +import unittest + +from mojom.generate import module as mojom + + +class ModuleTest(unittest.TestCase): + def testNonInterfaceAsInterfaceRequest(self): + """Tests that a non-interface cannot be used for interface requests.""" + module = mojom.Module('test_module', 'test_namespace') + struct = mojom.Struct('TestStruct', module=module) + with self.assertRaises(Exception) as e: + mojom.InterfaceRequest(struct) + self.assertEquals( + e.exception.__str__(), + 'Interface request requires \'x:TestStruct\' to be an interface.') + + def testNonInterfaceAsAssociatedInterface(self): + """Tests that a non-interface type cannot be used for associated interfaces. + """ + module = mojom.Module('test_module', 'test_namespace') + struct = mojom.Struct('TestStruct', module=module) + with self.assertRaises(Exception) as e: + mojom.AssociatedInterface(struct) + self.assertEquals( + e.exception.__str__(), + 'Associated interface requires \'x:TestStruct\' to be an interface.') diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/pack.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/pack.py new file mode 100644 index 00000000..61240426 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/pack.py @@ -0,0 +1,367 @@ +# Copyright 2013 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import copy +from mojom.generate import module as mojom + +# This module provides a mechanism for determining the packed order and offsets +# of a mojom.Struct. +# +# ps = pack.PackedStruct(struct) +# ps.packed_fields will access a list of PackedField objects, each of which +# will have an offset, a size and a bit (for mojom.BOOLs). + +# Size of struct header in bytes: num_bytes [4B] + version [4B]. +HEADER_SIZE = 8 + + +class PackedField: + kind_to_size = { + mojom.BOOL: 1, + mojom.INT8: 1, + mojom.UINT8: 1, + mojom.INT16: 2, + mojom.UINT16: 2, + mojom.INT32: 4, + mojom.UINT32: 4, + mojom.FLOAT: 4, + mojom.HANDLE: 4, + mojom.MSGPIPE: 4, + mojom.SHAREDBUFFER: 4, + mojom.PLATFORMHANDLE: 4, + mojom.DCPIPE: 4, + mojom.DPPIPE: 4, + mojom.NULLABLE_HANDLE: 4, + mojom.NULLABLE_MSGPIPE: 4, + mojom.NULLABLE_SHAREDBUFFER: 4, + mojom.NULLABLE_PLATFORMHANDLE: 4, + mojom.NULLABLE_DCPIPE: 4, + mojom.NULLABLE_DPPIPE: 4, + mojom.INT64: 8, + mojom.UINT64: 8, + mojom.DOUBLE: 8, + mojom.STRING: 8, + mojom.NULLABLE_STRING: 8 + } + + @classmethod + def GetSizeForKind(cls, kind): + if isinstance(kind, (mojom.Array, mojom.Map, mojom.Struct, mojom.Interface, + mojom.AssociatedInterface, mojom.PendingRemote, + mojom.PendingAssociatedRemote)): + return 8 + if isinstance(kind, mojom.Union): + return 16 + if isinstance(kind, (mojom.InterfaceRequest, mojom.PendingReceiver)): + kind = mojom.MSGPIPE + if isinstance( + kind, + (mojom.AssociatedInterfaceRequest, mojom.PendingAssociatedReceiver)): + return 4 + if isinstance(kind, mojom.Enum): + # TODO(mpcomplete): what about big enums? + return cls.kind_to_size[mojom.INT32] + if not kind in cls.kind_to_size: + raise Exception("Undefined type: %s. Did you forget to import the file " + "containing the definition?" % kind.spec) + return cls.kind_to_size[kind] + + @classmethod + def GetAlignmentForKind(cls, kind): + if isinstance(kind, (mojom.Interface, mojom.AssociatedInterface, + mojom.PendingRemote, mojom.PendingAssociatedRemote)): + return 4 + if isinstance(kind, mojom.Union): + return 8 + return cls.GetSizeForKind(kind) + + def __init__(self, + field, + index, + ordinal, + original_field=None, + sub_ordinal=None, + linked_value_packed_field=None): + """ + Args: + field: the original field. + index: the position of the original field in the struct. + ordinal: the ordinal of the field for serialization. + original_field: See below. + sub_ordinal: See below. + linked_value_packed_field: See below. + + original_field, sub_ordinal, and linked_value_packed_field are used to + support nullable ValueKind fields. For legacy reasons, nullable ValueKind + fields actually generate two PackedFields. This allows: + + - backwards compatibility prior to Mojo support for nullable ValueKinds. + - correct packing of fields for the aforementioned backwards compatibility. + + When translating Fields to PackedFields, the original field is turned into + two PackedFields: the first PackedField always has type mojom.BOOL, while + the second PackedField has the non-nullable version of the field's kind. + + When constructing these PackedFields, original_field references the field + as defined in the mojom; the name as defined in the mojom will be used for + all layers above the wire/data layer. + + sub_ordinal is used to sort the two PackedFields correctly with respect to + each other: the first mojom.BOOL field always has sub_ordinal 0, while the + second field always has sub_ordinal 1. + + Finally, linked_value_packed_field is used by the serialization and + deserialization helpers, which generally just iterate over a PackedStruct's + PackedField's in ordinal order. This allows the helpers to easily reference + any related PackedFields rather than having to lookup related PackedFields + by index while iterating. + """ + self.field = field + self.index = index + self.ordinal = ordinal + self.original_field = original_field + self.sub_ordinal = sub_ordinal + self.linked_value_packed_field = linked_value_packed_field + self.size = self.GetSizeForKind(self.field.kind) + self.alignment = self.GetAlignmentForKind(self.field.kind) + self.offset = None + self.bit = None + self.min_version = None + + +def GetPad(offset, alignment): + """Returns the pad necessary to reserve space so that |offset + pad| equals to + some multiple of |alignment|.""" + return (alignment - (offset % alignment)) % alignment + + +def GetFieldOffset(field, last_field): + """Returns a 2-tuple of the field offset and bit (for BOOLs).""" + if (field.field.kind == mojom.BOOL and last_field.field.kind == mojom.BOOL + and last_field.bit < 7): + return (last_field.offset, last_field.bit + 1) + + offset = last_field.offset + last_field.size + pad = GetPad(offset, field.alignment) + return (offset + pad, 0) + + +def GetPayloadSizeUpToField(field): + """Returns the payload size (not including struct header) if |field| is the + last field. + """ + if not field: + return 0 + offset = field.offset + field.size + pad = GetPad(offset, 8) + return offset + pad + + +def IsNullableValueKindPackedField(field): + """Returns true if `field` is derived from a nullable ValueKind field. + + Nullable ValueKind fields often require special handling in the bindings due + to the way the implementation is constrained for wire compatibility. + """ + assert isinstance(field, PackedField) + return field.sub_ordinal is not None + + +def IsPrimaryNullableValueKindPackedField(field): + """Returns true if `field` is derived from a nullable ValueKind mojom field + and is the "primary" field. + + The primary field is a bool PackedField that controls if the field should be + considered as present or not; it will have a reference to the PackedField that + holds the actual value representation if considered present. + + Bindings code that translates between the wire protocol and the higher layers + can use this to simplify mapping multiple PackedFields to the single field + that is logically exposed to bindings consumers. + """ + assert isinstance(field, PackedField) + return field.linked_value_packed_field is not None + + +class PackedStruct: + def __init__(self, struct): + self.struct = struct + # |packed_fields| contains all the fields, in increasing offset order. + self.packed_fields = [] + # |packed_fields_in_ordinal_order| refers to the same fields as + # |packed_fields|, but in ordinal order. + self.packed_fields_in_ordinal_order = [] + + # No fields. + if (len(struct.fields) == 0): + return + + # Start by sorting by ordinal. + src_fields = self.packed_fields_in_ordinal_order + ordinal = 0 + for index, field in enumerate(struct.fields): + if field.ordinal is not None: + ordinal = field.ordinal + # Nullable value types are a bit weird: they generate two PackedFields + # despite being a single ValueKind. This is for wire compatibility to + # ease the transition from legacy mojom syntax where nullable value types + # were not supported. + if isinstance(field.kind, mojom.ValueKind) and field.kind.is_nullable: + # The suffixes intentionally use Unicode codepoints which are considered + # valid C++/Java/JavaScript identifiers, yet are unlikely to be used in + # actual user code. + has_value_field = copy.copy(field) + has_value_field.name = f'{field.mojom_name}_$flag' + has_value_field.kind = mojom.BOOL + + value_field = copy.copy(field) + value_field.name = f'{field.mojom_name}_$value' + value_field.kind = field.kind.MakeUnnullableKind() + + value_packed_field = PackedField(value_field, + index, + ordinal, + original_field=field, + sub_ordinal=1, + linked_value_packed_field=None) + has_value_packed_field = PackedField( + has_value_field, + index, + ordinal, + original_field=field, + sub_ordinal=0, + linked_value_packed_field=value_packed_field) + src_fields.append(has_value_packed_field) + src_fields.append(value_packed_field) + else: + src_fields.append(PackedField(field, index, ordinal)) + ordinal += 1 + src_fields.sort(key=lambda field: (field.ordinal, field.sub_ordinal)) + + # Set |min_version| for each field. + next_min_version = 0 + for packed_field in src_fields: + if packed_field.field.min_version is None: + assert next_min_version == 0 + else: + assert packed_field.field.min_version >= next_min_version + next_min_version = packed_field.field.min_version + packed_field.min_version = next_min_version + + if (packed_field.min_version != 0 + and mojom.IsReferenceKind(packed_field.field.kind) + and not packed_field.field.kind.is_nullable): + raise Exception( + "Non-nullable reference fields are only allowed in version 0 of a " + "struct. %s.%s is defined with [MinVersion=%d]." % + (self.struct.name, packed_field.field.name, + packed_field.min_version)) + + src_field = src_fields[0] + src_field.offset = 0 + src_field.bit = 0 + dst_fields = self.packed_fields + dst_fields.append(src_field) + + # Then find first slot that each field will fit. + for src_field in src_fields[1:]: + last_field = dst_fields[0] + for i in range(1, len(dst_fields)): + next_field = dst_fields[i] + offset, bit = GetFieldOffset(src_field, last_field) + if offset + src_field.size <= next_field.offset: + # Found hole. + src_field.offset = offset + src_field.bit = bit + dst_fields.insert(i, src_field) + break + last_field = next_field + if src_field.offset is None: + # Add to end + src_field.offset, src_field.bit = GetFieldOffset(src_field, last_field) + dst_fields.append(src_field) + + +class ByteInfo: + def __init__(self): + self.is_padding = False + self.packed_fields = [] + + +def GetByteLayout(packed_struct): + total_payload_size = GetPayloadSizeUpToField( + packed_struct.packed_fields[-1] if packed_struct.packed_fields else None) + byte_info = [ByteInfo() for i in range(total_payload_size)] + + limit_of_previous_field = 0 + for packed_field in packed_struct.packed_fields: + for i in range(limit_of_previous_field, packed_field.offset): + byte_info[i].is_padding = True + byte_info[packed_field.offset].packed_fields.append(packed_field) + limit_of_previous_field = packed_field.offset + packed_field.size + + for i in range(limit_of_previous_field, len(byte_info)): + byte_info[i].is_padding = True + + for byte in byte_info: + # A given byte cannot both be padding and have a fields packed into it. + assert not (byte.is_padding and byte.packed_fields) + + return byte_info + + +class VersionInfo: + def __init__(self, version, num_fields, num_packed_fields, num_bytes): + self.version = version + self.num_fields = num_fields + self.num_packed_fields = num_packed_fields + self.num_bytes = num_bytes + + +def GetVersionInfo(packed_struct): + """Get version information for a struct. + + Args: + packed_struct: A PackedStruct instance. + + Returns: + A non-empty list of VersionInfo instances, sorted by version in increasing + order. + Note: The version numbers may not be consecutive. + """ + versions = [] + last_version = 0 + last_num_fields = 0 + last_num_packed_fields = 0 + last_payload_size = 0 + + for packed_field in packed_struct.packed_fields_in_ordinal_order: + if packed_field.min_version != last_version: + versions.append( + VersionInfo(last_version, last_num_fields, last_num_packed_fields, + last_payload_size + HEADER_SIZE)) + last_version = packed_field.min_version + + # Nullable numeric fields (e.g. `int32?`) expand to two packed fields, so to + # avoid double-counting, only increment if the field is: + # - not used for representing a nullable value kind field, or + # - the primary field representing the nullable value kind field. + last_num_fields += 1 if ( + not IsNullableValueKindPackedField(packed_field) + or IsPrimaryNullableValueKindPackedField(packed_field)) else 0 + + last_num_packed_fields += 1 + + # The fields are iterated in ordinal order here. However, the size of a + # version is determined by the last field of that version in pack order, + # instead of ordinal order. Therefore, we need to calculate the max value. + last_payload_size = max(GetPayloadSizeUpToField(packed_field), + last_payload_size) + + assert len( + versions) == 0 or last_num_packed_fields != versions[-1].num_packed_fields + versions.append( + VersionInfo(last_version, last_num_fields, last_num_packed_fields, + last_payload_size + HEADER_SIZE)) + return versions diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/pack_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/pack_unittest.py new file mode 100644 index 00000000..7d8e4e01 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/pack_unittest.py @@ -0,0 +1,253 @@ +# Copyright 2013 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import sys +import unittest + +from mojom.generate import module as mojom +from mojom.generate import pack + + +class PackTest(unittest.TestCase): + def testOrdinalOrder(self): + struct = mojom.Struct('test') + struct.AddField('testfield1', mojom.INT32, 2) + struct.AddField('testfield2', mojom.INT32, 1) + ps = pack.PackedStruct(struct) + + self.assertEqual(2, len(ps.packed_fields)) + self.assertEqual('testfield2', ps.packed_fields[0].field.mojom_name) + self.assertEqual('testfield1', ps.packed_fields[1].field.mojom_name) + + def testZeroFields(self): + struct = mojom.Struct('test') + ps = pack.PackedStruct(struct) + self.assertEqual(0, len(ps.packed_fields)) + + def testOneField(self): + struct = mojom.Struct('test') + struct.AddField('testfield1', mojom.INT8) + ps = pack.PackedStruct(struct) + self.assertEqual(1, len(ps.packed_fields)) + + def _CheckPackSequence(self, kinds, fields, offsets): + """Checks the pack order and offsets of a sequence of mojom.Kinds. + + Args: + kinds: A sequence of mojom.Kinds that specify the fields that are to be + created. + fields: The expected order of the resulting fields, with the integer "1" + first. + offsets: The expected order of offsets, with the integer "0" first. + """ + struct = mojom.Struct('test') + index = 1 + for kind in kinds: + struct.AddField('%d' % index, kind) + index += 1 + ps = pack.PackedStruct(struct) + num_fields = len(ps.packed_fields) + self.assertEqual(len(kinds), num_fields) + for i in range(num_fields): + self.assertEqual('%d' % fields[i], ps.packed_fields[i].field.mojom_name) + self.assertEqual(offsets[i], ps.packed_fields[i].offset) + + def testPaddingPackedInOrder(self): + return self._CheckPackSequence((mojom.INT8, mojom.UINT8, mojom.INT32), + (1, 2, 3), (0, 1, 4)) + + def testPaddingPackedOutOfOrder(self): + return self._CheckPackSequence((mojom.INT8, mojom.INT32, mojom.UINT8), + (1, 3, 2), (0, 1, 4)) + + def testPaddingPackedOverflow(self): + kinds = (mojom.INT8, mojom.INT32, mojom.INT16, mojom.INT8, mojom.INT8) + # 2 bytes should be packed together first, followed by short, then by int. + fields = (1, 4, 3, 2, 5) + offsets = (0, 1, 2, 4, 8) + return self._CheckPackSequence(kinds, fields, offsets) + + def testNullableTypes(self): + kinds = (mojom.STRING.MakeNullableKind(), mojom.HANDLE.MakeNullableKind(), + mojom.Struct('test_struct').MakeNullableKind(), + mojom.DCPIPE.MakeNullableKind(), mojom.Array().MakeNullableKind(), + mojom.DPPIPE.MakeNullableKind(), + mojom.Array(length=5).MakeNullableKind(), + mojom.MSGPIPE.MakeNullableKind(), + mojom.Interface('test_interface').MakeNullableKind(), + mojom.SHAREDBUFFER.MakeNullableKind(), + mojom.InterfaceRequest().MakeNullableKind()) + fields = (1, 2, 4, 3, 5, 6, 8, 7, 9, 10, 11) + offsets = (0, 8, 12, 16, 24, 32, 36, 40, 48, 56, 60) + return self._CheckPackSequence(kinds, fields, offsets) + + def testAllTypes(self): + return self._CheckPackSequence( + (mojom.BOOL, mojom.INT8, mojom.STRING, mojom.UINT8, mojom.INT16, + mojom.DOUBLE, mojom.UINT16, mojom.INT32, mojom.UINT32, mojom.INT64, + mojom.FLOAT, mojom.STRING, mojom.HANDLE, mojom.UINT64, + mojom.Struct('test'), mojom.Array(), mojom.STRING.MakeNullableKind()), + (1, 2, 4, 5, 7, 3, 6, 8, 9, 10, 11, 13, 12, 14, 15, 16, 17, 18), + (0, 1, 2, 4, 6, 8, 16, 24, 28, 32, 40, 44, 48, 56, 64, 72, 80, 88)) + + def testPaddingPackedOutOfOrderByOrdinal(self): + struct = mojom.Struct('test') + struct.AddField('testfield1', mojom.INT8) + struct.AddField('testfield3', mojom.UINT8, 3) + struct.AddField('testfield2', mojom.INT32, 2) + ps = pack.PackedStruct(struct) + self.assertEqual(3, len(ps.packed_fields)) + + # Second byte should be packed in behind first, altering order. + self.assertEqual('testfield1', ps.packed_fields[0].field.mojom_name) + self.assertEqual('testfield3', ps.packed_fields[1].field.mojom_name) + self.assertEqual('testfield2', ps.packed_fields[2].field.mojom_name) + + # Second byte should be packed with first. + self.assertEqual(0, ps.packed_fields[0].offset) + self.assertEqual(1, ps.packed_fields[1].offset) + self.assertEqual(4, ps.packed_fields[2].offset) + + def testBools(self): + struct = mojom.Struct('test') + struct.AddField('bit0', mojom.BOOL) + struct.AddField('bit1', mojom.BOOL) + struct.AddField('int', mojom.INT32) + struct.AddField('bit2', mojom.BOOL) + struct.AddField('bit3', mojom.BOOL) + struct.AddField('bit4', mojom.BOOL) + struct.AddField('bit5', mojom.BOOL) + struct.AddField('bit6', mojom.BOOL) + struct.AddField('bit7', mojom.BOOL) + struct.AddField('bit8', mojom.BOOL) + ps = pack.PackedStruct(struct) + self.assertEqual(10, len(ps.packed_fields)) + + # First 8 bits packed together. + for i in range(8): + pf = ps.packed_fields[i] + self.assertEqual(0, pf.offset) + self.assertEqual("bit%d" % i, pf.field.mojom_name) + self.assertEqual(i, pf.bit) + + # Ninth bit goes into second byte. + self.assertEqual("bit8", ps.packed_fields[8].field.mojom_name) + self.assertEqual(1, ps.packed_fields[8].offset) + self.assertEqual(0, ps.packed_fields[8].bit) + + # int comes last. + self.assertEqual("int", ps.packed_fields[9].field.mojom_name) + self.assertEqual(4, ps.packed_fields[9].offset) + + def testMinVersion(self): + """Tests that |min_version| is properly set for packed fields.""" + struct = mojom.Struct('test') + struct.AddField('field_2', mojom.BOOL, 2) + struct.AddField('field_0', mojom.INT32, 0) + struct.AddField('field_1', mojom.INT64, 1) + ps = pack.PackedStruct(struct) + + self.assertEqual('field_0', ps.packed_fields[0].field.mojom_name) + self.assertEqual('field_2', ps.packed_fields[1].field.mojom_name) + self.assertEqual('field_1', ps.packed_fields[2].field.mojom_name) + + self.assertEqual(0, ps.packed_fields[0].min_version) + self.assertEqual(0, ps.packed_fields[1].min_version) + self.assertEqual(0, ps.packed_fields[2].min_version) + + struct.fields[0].attributes = {'MinVersion': 1} + ps = pack.PackedStruct(struct) + + self.assertEqual(0, ps.packed_fields[0].min_version) + self.assertEqual(1, ps.packed_fields[1].min_version) + self.assertEqual(0, ps.packed_fields[2].min_version) + + def testGetVersionInfoEmptyStruct(self): + """Tests that pack.GetVersionInfo() never returns an empty list, even for + empty structs. + """ + struct = mojom.Struct('test') + ps = pack.PackedStruct(struct) + + versions = pack.GetVersionInfo(ps) + self.assertEqual(1, len(versions)) + self.assertEqual(0, versions[0].version) + self.assertEqual(0, versions[0].num_fields) + self.assertEqual(8, versions[0].num_bytes) + + def testGetVersionInfoComplexOrder(self): + """Tests pack.GetVersionInfo() using a struct whose definition order, + ordinal order and pack order for fields are all different. + """ + struct = mojom.Struct('test') + struct.AddField( + 'field_3', mojom.BOOL, ordinal=3, attributes={'MinVersion': 3}) + struct.AddField('field_0', mojom.INT32, ordinal=0) + struct.AddField( + 'field_1', mojom.INT64, ordinal=1, attributes={'MinVersion': 2}) + struct.AddField( + 'field_2', mojom.INT64, ordinal=2, attributes={'MinVersion': 3}) + ps = pack.PackedStruct(struct) + + versions = pack.GetVersionInfo(ps) + self.assertEqual(3, len(versions)) + + self.assertEqual(0, versions[0].version) + self.assertEqual(1, versions[0].num_fields) + self.assertEqual(16, versions[0].num_bytes) + + self.assertEqual(2, versions[1].version) + self.assertEqual(2, versions[1].num_fields) + self.assertEqual(24, versions[1].num_bytes) + + self.assertEqual(3, versions[2].version) + self.assertEqual(4, versions[2].num_fields) + self.assertEqual(32, versions[2].num_bytes) + + def testGetVersionInfoPackedStruct(self): + """Tests that pack.GetVersionInfo() correctly sets version, num_fields, + and num_packed_fields for a packed struct. + """ + struct = mojom.Struct('test') + struct.AddField('field_0', mojom.BOOL, ordinal=0) + struct.AddField('field_1', + mojom.NULLABLE_BOOL, + ordinal=1, + attributes={'MinVersion': 1}) + struct.AddField('field_2', + mojom.NULLABLE_BOOL, + ordinal=2, + attributes={'MinVersion': 2}) + ps = pack.PackedStruct(struct) + versions = pack.GetVersionInfo(ps) + + self.assertEqual(3, len(versions)) + self.assertEqual(0, versions[0].version) + self.assertEqual(1, versions[1].version) + self.assertEqual(2, versions[2].version) + self.assertEqual(1, versions[0].num_fields) + self.assertEqual(2, versions[1].num_fields) + self.assertEqual(3, versions[2].num_fields) + self.assertEqual(1, versions[0].num_packed_fields) + self.assertEqual(3, versions[1].num_packed_fields) + self.assertEqual(5, versions[2].num_packed_fields) + + def testInterfaceAlignment(self): + """Tests that interfaces are aligned on 4-byte boundaries, although the size + of an interface is 8 bytes. + """ + kinds = (mojom.INT32, mojom.Interface('test_interface')) + fields = (1, 2) + offsets = (0, 4) + self._CheckPackSequence(kinds, fields, offsets) + + def testAssociatedInterfaceAlignment(self): + """Tests that associated interfaces are aligned on 4-byte boundaries, + although the size of an associated interface is 8 bytes. + """ + kinds = (mojom.INT32, + mojom.AssociatedInterface(mojom.Interface('test_interface'))) + fields = (1, 2) + offsets = (0, 4) + self._CheckPackSequence(kinds, fields, offsets) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py new file mode 100644 index 00000000..807e2a4f --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/template_expander.py @@ -0,0 +1,82 @@ +# Copyright 2013 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Based on third_party/WebKit/Source/build/scripts/template_expander.py. + +import os.path +import sys + +from mojom import fileutil + +fileutil.AddLocalRepoThirdPartyDirToModulePath() +import jinja2 + + +def ApplyTemplate(mojo_generator, path_to_template, params, **kwargs): + loader = jinja2.ModuleLoader( + os.path.join(mojo_generator.bytecode_path, + "%s.zip" % mojo_generator.GetTemplatePrefix())) + final_kwargs = dict(mojo_generator.GetJinjaParameters()) + final_kwargs.update(kwargs) + + jinja_env = jinja2.Environment( + loader=loader, keep_trailing_newline=True, **final_kwargs) + jinja_env.globals.update(mojo_generator.GetGlobals()) + jinja_env.filters.update(mojo_generator.GetFilters()) + template = jinja_env.get_template(path_to_template) + return template.render(params) + + +def UseJinja(path_to_template, **kwargs): + def RealDecorator(generator): + def GeneratorInternal(*args, **kwargs2): + parameters = generator(*args, **kwargs2) + return ApplyTemplate(args[0], path_to_template, parameters, **kwargs) + + GeneratorInternal.__name__ = generator.__name__ + return GeneratorInternal + + return RealDecorator + + +def ApplyImportedTemplate(mojo_generator, path_to_template, filename, params, + **kwargs): + loader = jinja2.FileSystemLoader(searchpath=path_to_template) + final_kwargs = dict(mojo_generator.GetJinjaParameters()) + final_kwargs.update(kwargs) + + jinja_env = jinja2.Environment( + loader=loader, keep_trailing_newline=True, **final_kwargs) + jinja_env.globals.update(mojo_generator.GetGlobals()) + jinja_env.filters.update(mojo_generator.GetFilters()) + template = jinja_env.get_template(filename) + return template.render(params) + + +def UseJinjaForImportedTemplate(func): + def wrapper(*args, **kwargs): + parameters = func(*args, **kwargs) + path_to_template = args[1] + filename = args[2] + return ApplyImportedTemplate(args[0], path_to_template, filename, + parameters) + + wrapper.__name__ = func.__name__ + return wrapper + + +def PrecompileTemplates(generator_modules, output_dir): + for module in generator_modules.values(): + generator = module.Generator(None) + jinja_env = jinja2.Environment( + loader=jinja2.FileSystemLoader([ + os.path.join( + os.path.dirname(module.__file__), generator.GetTemplatePrefix()) + ])) + jinja_env.filters.update(generator.GetFilters()) + jinja_env.compile_templates(os.path.join( + output_dir, "%s.zip" % generator.GetTemplatePrefix()), + extensions=["tmpl"], + zip="stored", + ignore_errors=False) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/translate.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/translate.py new file mode 100644 index 00000000..83bb297f --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/translate.py @@ -0,0 +1,1258 @@ +# Copyright 2013 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Convert parse tree to AST. + +This module converts the parse tree to the AST we use for code generation. The +main entry point is OrderedModule, which gets passed the parser +representation of a mojom file. When called it's assumed that all imports have +already been parsed and converted to ASTs before. +""" + +import itertools +import os +import re + +from collections import OrderedDict +from mojom.generate import generator +from mojom.generate import module as mojom +from mojom.parse import ast + + +is_running_backwards_compatibility_check_hack = False + +### DO NOT ADD ENTRIES TO THIS LIST. ### +_EXTENSIBLE_ENUMS_MISSING_DEFAULT = ( + 'x:arc.keymaster.mojom.Algorithm', + 'x:arc.keymaster.mojom.Digest', + 'x:arc.keymaster.mojom.SignatureResult', + 'x:arc.mojom.AccessibilityActionType', + 'x:arc.mojom.AccessibilityBooleanProperty', + 'x:arc.mojom.AccessibilityEventIntListProperty', + 'x:arc.mojom.AccessibilityEventIntProperty', + 'x:arc.mojom.AccessibilityEventStringProperty', + 'x:arc.mojom.AccessibilityEventType', + 'x:arc.mojom.AccessibilityFilterType', + 'x:arc.mojom.AccessibilityIntListProperty', + 'x:arc.mojom.AccessibilityIntProperty', + 'x:arc.mojom.AccessibilityLiveRegionType', + 'x:arc.mojom.AccessibilityNotificationStateType', + 'x:arc.mojom.AccessibilityRangeType', + 'x:arc.mojom.AccessibilitySelectionMode', + 'x:arc.mojom.AccessibilityStringListProperty', + 'x:arc.mojom.AccessibilityStringProperty', + 'x:arc.mojom.AccessibilityWindowBooleanProperty', + 'x:arc.mojom.AccessibilityWindowIntListProperty', + 'x:arc.mojom.AccessibilityWindowIntProperty', + 'x:arc.mojom.AccessibilityWindowStringProperty', + 'x:arc.mojom.AccessibilityWindowType', + 'x:arc.mojom.AccountCheckStatus', + 'x:arc.mojom.AccountUpdateType', + 'x:arc.mojom.ActionType', + 'x:arc.mojom.Algorithm', + 'x:arc.mojom.AndroidIdSource', + 'x:arc.mojom.AnrSource', + 'x:arc.mojom.AnrType', + 'x:arc.mojom.AppDiscoveryRequestState', + 'x:arc.mojom.AppKillType', + 'x:arc.mojom.AppPermission', + 'x:arc.mojom.AppPermissionGroup', + 'x:arc.mojom.AppReinstallState', + 'x:arc.mojom.AppShortcutItemType', + 'x:arc.mojom.ArcAuthCodeStatus', + 'x:arc.mojom.ArcClipboardDragDropEvent', + 'x:arc.mojom.ArcCorePriAbiMigEvent', + 'x:arc.mojom.ArcDnsQuery', + 'x:arc.mojom.ArcImageCopyPasteCompatAction', + 'x:arc.mojom.ArcNetworkError', + 'x:arc.mojom.ArcNetworkEvent', + 'x:arc.mojom.ArcNotificationEvent', + 'x:arc.mojom.ArcNotificationExpandState', + 'x:arc.mojom.ArcNotificationPriority', + 'x:arc.mojom.ArcNotificationRemoteInputState', + 'x:arc.mojom.ArcNotificationShownContents', + 'x:arc.mojom.ArcNotificationStyle', + 'x:arc.mojom.ArcNotificationType', + 'x:arc.mojom.ArcPipEvent', + 'x:arc.mojom.ArcResizeLockState', + 'x:arc.mojom.ArcSignInSuccess', + 'x:arc.mojom.ArcTimerResult', + 'x:arc.mojom.AudioSwitch', + 'x:arc.mojom.BluetoothAclState', + 'x:arc.mojom.BluetoothAdapterState', + 'x:arc.mojom.BluetoothAdvertisingDataType', + 'x:arc.mojom.BluetoothBondState', + 'x:arc.mojom.BluetoothDeviceType', + 'x:arc.mojom.BluetoothDiscoveryState', + 'x:arc.mojom.BluetoothGattDBAttributeType', + 'x:arc.mojom.BluetoothGattStatus', + 'x:arc.mojom.BluetoothPropertyType', + 'x:arc.mojom.BluetoothScanMode', + 'x:arc.mojom.BluetoothSdpAttributeType', + 'x:arc.mojom.BluetoothSocketType', + 'x:arc.mojom.BluetoothStatus', + 'x:arc.mojom.BootType', + 'x:arc.mojom.CaptionTextShadowType', + 'x:arc.mojom.ChangeType', + 'x:arc.mojom.ChromeAccountType', + 'x:arc.mojom.ChromeApp', + 'x:arc.mojom.ChromePage', + 'x:arc.mojom.ClockId', + 'x:arc.mojom.CloudProvisionFlowError', + 'x:arc.mojom.CommandResultType', + 'x:arc.mojom.CompanionLibApiId', + 'x:arc.mojom.ConnectionStateType', + 'x:arc.mojom.ContentChangeType', + 'x:arc.mojom.CpuRestrictionState', + 'x:arc.mojom.CursorCoordinateSpace', + 'x:arc.mojom.DataRestoreStatus', + 'x:arc.mojom.DecoderStatus', + 'x:arc.mojom.DeviceType', + 'x:arc.mojom.Digest', + 'x:arc.mojom.DisplayWakeLockType', + 'x:arc.mojom.EapMethod', + 'x:arc.mojom.EapPhase2Method', + 'x:arc.mojom.FileSelectorEventType', + 'x:arc.mojom.GMSCheckInError', + 'x:arc.mojom.GMSSignInError', + 'x:arc.mojom.GeneralSignInError', + 'x:arc.mojom.GetNetworksRequestType', + 'x:arc.mojom.HalPixelFormat', + 'x:arc.mojom.IPAddressType', + 'x:arc.mojom.InstallErrorReason', + 'x:arc.mojom.KeyFormat', + 'x:arc.mojom.KeyManagement', + 'x:arc.mojom.KeyPurpose', + 'x:arc.mojom.KeymasterError', + 'x:arc.mojom.MainAccountHashMigrationStatus', + 'x:arc.mojom.MainAccountResolutionStatus', + 'x:arc.mojom.ManagementChangeStatus', + 'x:arc.mojom.ManagementState', + 'x:arc.mojom.MessageCenterVisibility', + 'x:arc.mojom.MetricsType', + 'x:arc.mojom.MountEvent', + 'x:arc.mojom.NativeBridgeType', + 'x:arc.mojom.NetworkResult', + 'x:arc.mojom.NetworkType', + 'x:arc.mojom.OemCryptoAlgorithm', + 'x:arc.mojom.OemCryptoCipherMode', + 'x:arc.mojom.OemCryptoHdcpCapability', + 'x:arc.mojom.OemCryptoLicenseType', + 'x:arc.mojom.OemCryptoPrivateKey', + 'x:arc.mojom.OemCryptoProvisioningMethod', + 'x:arc.mojom.OemCryptoResult', + 'x:arc.mojom.OemCryptoRsaPaddingScheme', + 'x:arc.mojom.OemCryptoUsageEntryStatus', + 'x:arc.mojom.Padding', + 'x:arc.mojom.PaiFlowState', + 'x:arc.mojom.PatternType', + 'x:arc.mojom.PressureLevel', + 'x:arc.mojom.PrintColorMode', + 'x:arc.mojom.PrintContentType', + 'x:arc.mojom.PrintDuplexMode', + 'x:arc.mojom.PrinterStatus', + 'x:arc.mojom.ProcessState', + 'x:arc.mojom.PurchaseState', + 'x:arc.mojom.ReauthReason', + 'x:arc.mojom.ScaleFactor', + 'x:arc.mojom.SecurityType', + 'x:arc.mojom.SegmentStyle', + 'x:arc.mojom.SelectFilesActionType', + 'x:arc.mojom.SetNativeChromeVoxResponse', + 'x:arc.mojom.ShowPackageInfoPage', + 'x:arc.mojom.SpanType', + 'x:arc.mojom.SupportedLinkChangeSource', + 'x:arc.mojom.TetheringClientState', + 'x:arc.mojom.TextInputType', + 'x:arc.mojom.TtsEventType', + 'x:arc.mojom.VideoCodecProfile', + 'x:arc.mojom.VideoDecodeAccelerator.Result', + 'x:arc.mojom.VideoEncodeAccelerator.Error', + 'x:arc.mojom.VideoFrameStorageType', + 'x:arc.mojom.VideoPixelFormat', + 'x:arc.mojom.WakefulnessMode', + 'x:arc.mojom.WebApkInstallResult', + 'x:ash.ime.mojom.InputFieldType', + 'x:ash.ime.mojom.PersonalizationMode', + 'x:ash.language.mojom.FeatureId', + 'x:blink.mojom.ScrollRestorationType', + 'x:chromeos.cdm.mojom.CdmKeyStatus', + 'x:chromeos.cdm.mojom.CdmMessageType', + 'x:chromeos.cdm.mojom.CdmSessionType', + 'x:chromeos.cdm.mojom.DecryptStatus', + 'x:chromeos.cdm.mojom.EmeInitDataType', + 'x:chromeos.cdm.mojom.EncryptionScheme', + 'x:chromeos.cdm.mojom.HdcpVersion', + 'x:chromeos.cdm.mojom.OutputProtection.LinkType', + 'x:chromeos.cdm.mojom.OutputProtection.ProtectionType', + 'x:chromeos.cdm.mojom.PromiseException', + 'x:chromeos.cfm.mojom.EnqueuePriority', + 'x:chromeos.cfm.mojom.LoggerErrorCode', + 'x:chromeos.cfm.mojom.LoggerState', + 'x:chromeos.cros_healthd.mojom.CryptoAlgorithm', + 'x:chromeos.cros_healthd.mojom.EncryptionState', + 'x:chromeos.machine_learning.mojom.AnnotationUsecase', + 'x:chromeos.machine_learning.mojom.BuiltinModelId', + 'x:chromeos.machine_learning.mojom.CreateGraphExecutorResult', + 'x:chromeos.machine_learning.mojom.DocumentScannerResultStatus', + 'x:chromeos.machine_learning.mojom.EndpointReason', + 'x:chromeos.machine_learning.mojom.EndpointerType', + 'x:chromeos.machine_learning.mojom.ExecuteResult', + 'x:chromeos.machine_learning.mojom.GrammarCheckerResult.Status', + 'x:chromeos.machine_learning.mojom.HandwritingRecognizerResult.Status', + 'x:chromeos.machine_learning.mojom.LoadHandwritingModelResult', + 'x:chromeos.machine_learning.mojom.LoadModelResult', + 'x:chromeos.machine_learning.mojom.Rotation', + 'x:chromeos.network_config.mojom.ConnectionStateType', + 'x:chromeos.network_config.mojom.DeviceStateType', + 'x:chromeos.network_config.mojom.IPConfigType', + 'x:chromeos.network_config.mojom.NetworkType', + 'x:chromeos.network_config.mojom.OncSource', + 'x:chromeos.network_config.mojom.PolicySource', + 'x:chromeos.network_config.mojom.PortalState', + 'x:chromeos.wilco_dtc_supportd.mojom.WilcoDtcSupportdEvent', + 'x:chromeos.wilco_dtc_supportd.mojom.WilcoDtcSupportdWebRequestHttpMethod', + 'x:chromeos.wilco_dtc_supportd.mojom.WilcoDtcSupportdWebRequestStatus', + 'x:cros.mojom.CameraClientType', + 'x:cros.mojom.CameraMetadataSectionStart', + 'x:cros.mojom.CameraMetadataTag', + 'x:cros.mojom.HalPixelFormat', + 'x:crosapi.mojom.AllowedPaths', + 'x:crosapi.mojom.BrowserAppInstanceType', + 'x:crosapi.mojom.CreationResult', + 'x:crosapi.mojom.DeviceAccessResultCode', + 'x:crosapi.mojom.DeviceMode', + 'x:crosapi.mojom.DlpRestrictionLevel', + 'x:crosapi.mojom.ExoImeSupport', + 'x:crosapi.mojom.FullscreenVisibility', + 'x:crosapi.mojom.GoogleServiceAuthError.State', + 'x:crosapi.mojom.IsInstallableResult', + 'x:crosapi.mojom.KeyTag', + 'x:crosapi.mojom.KeystoreSigningAlgorithmName', + 'x:crosapi.mojom.KeystoreType', + 'x:crosapi.mojom.LacrosFeedbackSource', + 'x:crosapi.mojom.MemoryPressureLevel', + 'x:crosapi.mojom.MetricsReportingManaged', + 'x:crosapi.mojom.NotificationType', + 'x:crosapi.mojom.OndeviceHandwritingSupport', + 'x:crosapi.mojom.OpenResult', + 'x:crosapi.mojom.PolicyDomain', + 'x:crosapi.mojom.RegistrationCodeType', + 'x:crosapi.mojom.ScaleFactor', + 'x:crosapi.mojom.SearchResult.OptionalBool', + 'x:crosapi.mojom.SelectFileDialogType', + 'x:crosapi.mojom.SelectFileResult', + 'x:crosapi.mojom.SharesheetResult', + 'x:crosapi.mojom.TouchEventType', + 'x:crosapi.mojom.VideoRotation', + 'x:crosapi.mojom.WallpaperLayout', + 'x:crosapi.mojom.WebAppInstallResultCode', + 'x:crosapi.mojom.WebAppUninstallResultCode', + 'x:device.mojom.HidBusType', + 'x:device.mojom.WakeLockReason', + 'x:device.mojom.WakeLockType', + 'x:drivefs.mojom.DialogReason.Type', + 'x:drivefs.mojom.DriveError.Type', + 'x:drivefs.mojom.DriveFsDelegate.ExtensionConnectionStatus', + 'x:drivefs.mojom.FileMetadata.CanPinStatus', + 'x:drivefs.mojom.FileMetadata.Type', + 'x:drivefs.mojom.ItemEventReason', + 'x:drivefs.mojom.MirrorPathStatus', + 'x:drivefs.mojom.MirrorSyncStatus', + 'x:drivefs.mojom.QueryParameters.SortField', + 'x:fuzz.mojom.FuzzEnum', + 'x:media.mojom.FillLightMode', + 'x:media.mojom.MeteringMode', + 'x:media.mojom.PowerLineFrequency', + 'x:media.mojom.RedEyeReduction', + 'x:media.mojom.ResolutionChangePolicy', + 'x:media.mojom.VideoCaptureApi', + 'x:media.mojom.VideoCaptureBufferType', + 'x:media.mojom.VideoCaptureError', + 'x:media.mojom.VideoCaptureFrameDropReason', + 'x:media.mojom.VideoCapturePixelFormat', + 'x:media.mojom.VideoCaptureTransportType', + 'x:media.mojom.VideoFacingMode', + 'x:media_session.mojom.AudioFocusType', + 'x:media_session.mojom.CameraState', + 'x:media_session.mojom.EnforcementMode', + 'x:media_session.mojom.MediaAudioVideoState', + 'x:media_session.mojom.MediaImageBitmapColorType', + 'x:media_session.mojom.MediaPictureInPictureState', + 'x:media_session.mojom.MediaPlaybackState', + 'x:media_session.mojom.MediaSession.SuspendType', + 'x:media_session.mojom.MediaSessionAction', + 'x:media_session.mojom.MediaSessionImageType', + 'x:media_session.mojom.MediaSessionInfo.SessionState', + 'x:media_session.mojom.MicrophoneState', + 'x:ml.model_loader.mojom.ComputeResult', + 'x:ml.model_loader.mojom.CreateModelLoaderResult', + 'x:ml.model_loader.mojom.LoadModelResult', + 'x:mojo.test.AnExtensibleEnum', + 'x:mojo.test.EnumB', + 'x:mojo.test.ExtensibleEmptyEnum', + 'x:mojo.test.enum_default_unittest.mojom.ExtensibleEnumWithoutDefault', + 'x:network.mojom.WebSandboxFlags', + 'x:payments.mojom.BillingResponseCode', + 'x:payments.mojom.CreateDigitalGoodsResponseCode', + 'x:payments.mojom.ItemType', + 'x:printing.mojom.PrinterType', + 'x:ui.mojom.KeyboardCode', +) +### DO NOT ADD ENTRIES TO THIS LIST. ### + + +def _DuplicateName(values): + """Returns the 'mojom_name' of the first entry in |values| whose 'mojom_name' + has already been encountered. If there are no duplicates, returns None.""" + names = set() + for value in values: + if value.mojom_name in names: + return value.mojom_name + names.add(value.mojom_name) + return None + + +def _ElemsOfType(elems, elem_type, scope): + """Find all elements of the given type. + + Args: + elems: {Sequence[Any]} Sequence of elems. + elem_type: {Type[C]} Extract all elems of this type. + scope: {str} The name of the surrounding scope (e.g. struct + definition). Used in error messages. + + Returns: + {List[C]} All elems of matching type. + """ + assert isinstance(elem_type, type) + result = [elem for elem in elems if isinstance(elem, elem_type)] + duplicate_name = _DuplicateName(result) + if duplicate_name: + raise Exception('Names in mojom must be unique within a scope. The name ' + '"%s" is used more than once within the scope "%s".' % + (duplicate_name, scope)) + return result + + +def _ProcessElements(scope, elements, operations_by_type): + """Iterates over the given elements, running a function from + operations_by_type for any element that matches a key in that dict. The scope + is the name of the surrounding scope, such as a filename or struct name, used + only in error messages.""" + names_in_this_scope = set() + for element in elements: + # pylint: disable=unidiomatic-typecheck + element_type = type(element) + if element_type in operations_by_type: + if element.mojom_name in names_in_this_scope: + raise Exception('Names must be unique within a scope. The name "%s" is ' + 'used more than once within the scope "%s".' % + (duplicate_name, scope)) + operations_by_type[element_type](element) + + +def _MapKind(kind): + map_to_kind = { + 'bool': 'b', + 'int8': 'i8', + 'int16': 'i16', + 'int32': 'i32', + 'int64': 'i64', + 'uint8': 'u8', + 'uint16': 'u16', + 'uint32': 'u32', + 'uint64': 'u64', + 'float': 'f', + 'double': 'd', + 'string': 's', + 'handle': 'h', + 'handle': 'h:d:c', + 'handle': 'h:d:p', + 'handle': 'h:m', + 'handle': 'h:s', + 'handle': 'h:p' + } + if kind.endswith('?'): + base_kind = _MapKind(kind[0:-1]) + return '?' + base_kind + if kind.endswith('}'): + lbracket = kind.rfind('{') + value = kind[0:lbracket] + return 'm[' + _MapKind(kind[lbracket + 1:-1]) + '][' + _MapKind(value) + ']' + if kind.endswith(']'): + lbracket = kind.rfind('[') + typename = kind[0:lbracket] + return 'a' + kind[lbracket + 1:-1] + ':' + _MapKind(typename) + if kind.startswith('asso<'): + assert kind.endswith('>') + return 'asso:' + _MapKind(kind[5:-1]) + if kind.startswith('rmt<'): + assert kind.endswith('>') + return 'rmt:' + _MapKind(kind[4:-1]) + if kind.startswith('rcv<'): + assert kind.endswith('>') + return 'rcv:' + _MapKind(kind[4:-1]) + if kind.startswith('rma<'): + assert kind.endswith('>') + return 'rma:' + _MapKind(kind[4:-1]) + if kind.startswith('rca<'): + assert kind.endswith('>') + return 'rca:' + _MapKind(kind[4:-1]) + if kind in map_to_kind: + return map_to_kind[kind] + return 'x:' + kind + + +def _MapAttributeValue(module, kind, value): + # True/False/None + if value is None: + return value + if not isinstance(value, str): + return value + # Is the attribute value the name of a feature? + try: + # Features cannot be nested in other types, so lookup in the global scope. + trial = _LookupKind(module.kinds, 'x:' + value, + _GetScopeForKind(module, kind)) + if isinstance(trial, mojom.Feature): + return trial + except ValueError: + pass + # Is the attribute value a constant or enum value? + try: + trial = _LookupValue(module, None, None, ('IDENTIFIER', value)) + if isinstance(trial, mojom.ConstantValue): + return trial.constant + if isinstance(trial, mojom.EnumValue): + return trial + except ValueError: + pass + # If not a referenceable mojo type - return as a string. + return value + + +def _AttributeListToDict(module, kind, attribute_list): + if attribute_list is None: + return None + assert isinstance(attribute_list, ast.AttributeList) + attributes = dict() + for attribute in attribute_list: + if attribute.key in attributes: + raise Exception("Duplicate key (%s) in attribute list" % attribute.key) + attributes[attribute.key] = _MapAttributeValue(module, kind, + attribute.value) + return attributes + + +builtin_values = frozenset([ + "double.INFINITY", "double.NEGATIVE_INFINITY", "double.NAN", + "float.INFINITY", "float.NEGATIVE_INFINITY", "float.NAN" +]) + + +def _IsBuiltinValue(value): + return value in builtin_values + + +def _LookupKind(kinds, spec, scope): + """Tries to find which Kind a spec refers to, given the scope in which its + referenced. Starts checking from the narrowest scope to most general. For + example, given a struct field like + Foo.Bar x; + Foo.Bar could refer to the type 'Bar' in the 'Foo' namespace, or an inner + type 'Bar' in the struct 'Foo' in the current namespace. + + |scope| is a tuple that looks like (namespace, struct/interface), referring + to the location where the type is referenced.""" + if spec.startswith('x:'): + mojom_name = spec[2:] + for i in range(len(scope), -1, -1): + test_spec = 'x:' + if i > 0: + test_spec += '.'.join(scope[:i]) + '.' + test_spec += mojom_name + kind = kinds.get(test_spec) + if kind: + return kind + + return kinds.get(spec) + + +def _GetScopeForKind(module, kind): + """For a given kind, returns a tuple of progressively more specific names + used to qualify the kind. For example if kind is an enum named Bar nested in a + struct Foo within module 'foo', this would return ('foo', 'Foo', 'Bar')""" + if isinstance(kind, mojom.Enum) and kind.parent_kind: + # Enums may be nested in other kinds. + return _GetScopeForKind(module, kind.parent_kind) + (kind.mojom_name, ) + + module_fragment = (module.mojom_namespace, ) if module.mojom_namespace else () + kind_fragment = (kind.mojom_name, ) if kind else () + return module_fragment + kind_fragment + + +def _LookupValueInScope(module, kind, identifier): + """Given a kind and an identifier, this attempts to resolve the given + identifier to a concrete NamedValue within the scope of the given kind.""" + scope = _GetScopeForKind(module, kind) + for i in reversed(range(len(scope) + 1)): + qualified_name = '.'.join(scope[:i] + (identifier, )) + value = module.values.get(qualified_name) + if value: + return value + return None + + +def _LookupValue(module, parent_kind, implied_kind, ast_leaf_node): + """Resolves a leaf node in the form ('IDENTIFIER', 'x') to a constant value + identified by 'x' in some mojom definition. parent_kind is used as context + when resolving the identifier. If the given leaf node is not an IDENTIFIER + (e.g. already a constant value), it is returned as-is. + + If implied_kind is provided, the parsed identifier may also be resolved within + its scope as fallback. This can be useful for more concise value references + when assigning enum-typed constants or field values.""" + if not isinstance(ast_leaf_node, tuple) or ast_leaf_node[0] != 'IDENTIFIER': + return ast_leaf_node + + # First look for a known user-defined identifier to resolve this within the + # enclosing scope. + identifier = ast_leaf_node[1] + + value = _LookupValueInScope(module, parent_kind, identifier) + if value: + return value + + # Next look in the scope of implied_kind, if provided. + value = (implied_kind and implied_kind.module and _LookupValueInScope( + implied_kind.module, implied_kind, identifier)) + if value: + return value + + # Fall back on defined builtin symbols + if _IsBuiltinValue(identifier): + return mojom.BuiltinValue(identifier) + + raise ValueError('Unknown identifier %s' % identifier) + + +def _Kind(kinds, spec, scope): + """Convert a type name into a mojom.Kind object. + + As a side-effect this function adds the result to 'kinds'. + + Args: + kinds: {Dict[str, mojom.Kind]} All known kinds up to this point, indexed by + their names. + spec: {str} A name uniquely identifying a type. + scope: {Tuple[str, str]} A tuple that looks like (namespace, + struct/interface), referring to the location where the type is + referenced. + + Returns: + {mojom.Kind} The type corresponding to 'spec'. + """ + kind = _LookupKind(kinds, spec, scope) + if kind: + return kind + + if spec.startswith('?'): + kind = _Kind(kinds, spec[1:], scope) + kind = kind.MakeNullableKind() + elif spec.startswith('a:'): + kind = mojom.Array(_Kind(kinds, spec[2:], scope)) + elif spec.startswith('asso:'): + inner_kind = _Kind(kinds, spec[5:], scope) + if isinstance(inner_kind, mojom.InterfaceRequest): + kind = mojom.AssociatedInterfaceRequest(inner_kind) + else: + kind = mojom.AssociatedInterface(inner_kind) + elif spec.startswith('a'): + colon = spec.find(':') + length = int(spec[1:colon]) + kind = mojom.Array(_Kind(kinds, spec[colon + 1:], scope), length) + elif spec.startswith('r:'): + kind = mojom.InterfaceRequest(_Kind(kinds, spec[2:], scope)) + elif spec.startswith('rmt:'): + kind = mojom.PendingRemote(_Kind(kinds, spec[4:], scope)) + elif spec.startswith('rcv:'): + kind = mojom.PendingReceiver(_Kind(kinds, spec[4:], scope)) + elif spec.startswith('rma:'): + kind = mojom.PendingAssociatedRemote(_Kind(kinds, spec[4:], scope)) + elif spec.startswith('rca:'): + kind = mojom.PendingAssociatedReceiver(_Kind(kinds, spec[4:], scope)) + elif spec.startswith('m['): + # Isolate the two types from their brackets. + + # It is not allowed to use map as key, so there shouldn't be nested ']'s + # inside the key type spec. + key_end = spec.find(']') + assert key_end != -1 and key_end < len(spec) - 1 + assert spec[key_end + 1] == '[' and spec[-1] == ']' + + first_kind = spec[2:key_end] + second_kind = spec[key_end + 2:-1] + + kind = mojom.Map( + _Kind(kinds, first_kind, scope), _Kind(kinds, second_kind, scope)) + else: + kind = mojom.Kind(spec) + + kinds[spec] = kind + return kind + + +def _Import(module, import_module): + # Copy the struct kinds from our imports into the current module. + importable_kinds = (mojom.Struct, mojom.Union, mojom.Enum, mojom.Interface, + mojom.Feature) + for kind in import_module.kinds.values(): + if (isinstance(kind, importable_kinds) + and kind.module.path == import_module.path): + module.kinds[kind.spec] = kind + # Ditto for values. + for value in import_module.values.values(): + if value.module.path == import_module.path: + module.values[value.GetSpec()] = value + + return import_module + + +def _Feature(module, parsed_feature): + """ + Args: + module: {mojom.Module} Module currently being constructed. + parsed_feature: {ast.Feature} Parsed feature. + + Returns: + {mojom.Feature} AST feature. + """ + feature = mojom.Feature(module=module) + feature.mojom_name = parsed_feature.mojom_name + feature.spec = 'x:' + module.GetNamespacePrefix() + feature.mojom_name + module.kinds[feature.spec] = feature + feature.constants = [] + _ProcessElements( + parsed_feature.mojom_name, parsed_feature.body, { + ast.Const: + lambda const: feature.constants.append( + _Constant(module, const, feature)), + }) + + feature.attributes = _AttributeListToDict(module, feature, + parsed_feature.attribute_list) + return feature + + +def _Struct(module, parsed_struct): + """ + Args: + module: {mojom.Module} Module currently being constructed. + parsed_struct: {ast.Struct} Parsed struct. + + Returns: + {mojom.Struct} AST struct. + """ + struct = mojom.Struct(module=module) + struct.mojom_name = parsed_struct.mojom_name + struct.native_only = parsed_struct.body is None + struct.spec = 'x:' + module.GetNamespacePrefix() + struct.mojom_name + module.kinds[struct.spec] = struct + struct.enums = [] + struct.constants = [] + struct.fields_data = [] + if not struct.native_only: + _ProcessElements( + parsed_struct.mojom_name, parsed_struct.body, { + ast.Enum: + lambda enum: struct.enums.append(_Enum(module, enum, struct)), + ast.Const: + lambda const: struct.constants.append( + _Constant(module, const, struct)), + ast.StructField: + struct.fields_data.append, + }) + + struct.attributes = _AttributeListToDict(module, struct, + parsed_struct.attribute_list) + + # Enforce that a [Native] attribute is set to make native-only struct + # declarations more explicit. + if struct.native_only: + if not struct.attributes or not struct.attributes.get('Native', False): + raise Exception("Native-only struct declarations must include a " + + "Native attribute.") + + if struct.attributes and struct.attributes.get('CustomSerializer', False): + struct.custom_serializer = True + + return struct + + +def _Union(module, parsed_union): + """ + Args: + module: {mojom.Module} Module currently being constructed. + parsed_union: {ast.Union} Parsed union. + + Returns: + {mojom.Union} AST union. + """ + union = mojom.Union(module=module) + union.mojom_name = parsed_union.mojom_name + union.spec = 'x:' + module.GetNamespacePrefix() + union.mojom_name + module.kinds[union.spec] = union + # Stash fields parsed_union here temporarily. + union.fields_data = [] + _ProcessElements(parsed_union.mojom_name, parsed_union.body, + {ast.UnionField: union.fields_data.append}) + union.attributes = _AttributeListToDict(module, union, + parsed_union.attribute_list) + return union + + +def _StructField(module, parsed_field, struct): + """ + Args: + module: {mojom.Module} Module currently being constructed. + parsed_field: {ast.StructField} Parsed struct field. + struct: {mojom.Struct} Struct this field belongs to. + + Returns: + {mojom.StructField} AST struct field. + """ + field = mojom.StructField() + field.mojom_name = parsed_field.mojom_name + field.kind = _Kind(module.kinds, _MapKind(parsed_field.typename), + (module.mojom_namespace, struct.mojom_name)) + field.ordinal = parsed_field.ordinal.value if parsed_field.ordinal else None + field.default = _LookupValue(module, struct, field.kind, + parsed_field.default_value) + field.attributes = _AttributeListToDict(module, field, + parsed_field.attribute_list) + return field + + +def _UnionField(module, parsed_field, union): + """ + Args: + module: {mojom.Module} Module currently being constructed. + parsed_field: {ast.UnionField} Parsed union field. + union: {mojom.Union} Union this fields belong to. + + Returns: + {mojom.UnionField} AST union. + """ + field = mojom.UnionField() + field.mojom_name = parsed_field.mojom_name + # Disallow unions from being self-recursive. + parsed_typename = parsed_field.typename + if parsed_typename.endswith('?'): + parsed_typename = parsed_typename[:-1] + assert parsed_typename != union.mojom_name + field.kind = _Kind(module.kinds, _MapKind(parsed_field.typename), + (module.mojom_namespace, union.mojom_name)) + field.ordinal = parsed_field.ordinal.value if parsed_field.ordinal else None + field.default = None + field.attributes = _AttributeListToDict(module, field, + parsed_field.attribute_list) + if field.is_default and not mojom.IsNullableKind(field.kind) and \ + not mojom.IsIntegralKind(field.kind): + raise Exception( + '[Default] field for union %s must be nullable or integral type.' % + union.mojom_name) + return field + + +def _Parameter(module, parsed_param, interface): + """ + Args: + module: {mojom.Module} Module currently being constructed. + parsed_param: {ast.Parameter} Parsed parameter. + union: {mojom.Interface} Interface this parameter belongs to. + + Returns: + {mojom.Parameter} AST parameter. + """ + parameter = mojom.Parameter() + parameter.mojom_name = parsed_param.mojom_name + parameter.kind = _Kind(module.kinds, _MapKind(parsed_param.typename), + (module.mojom_namespace, interface.mojom_name)) + parameter.ordinal = (parsed_param.ordinal.value + if parsed_param.ordinal else None) + parameter.default = None # TODO(tibell): We never have these. Remove field? + parameter.attributes = _AttributeListToDict(module, parameter, + parsed_param.attribute_list) + return parameter + + +def _Method(module, parsed_method, interface): + """ + Args: + module: {mojom.Module} Module currently being constructed. + parsed_method: {ast.Method} Parsed method. + interface: {mojom.Interface} Interface this method belongs to. + + Returns: + {mojom.Method} AST method. + """ + method = mojom.Method( + interface, + parsed_method.mojom_name, + ordinal=parsed_method.ordinal.value if parsed_method.ordinal else None) + method.parameters = list( + map(lambda parameter: _Parameter(module, parameter, interface), + parsed_method.parameter_list)) + if parsed_method.response_parameter_list is not None: + method.response_parameters = list( + map(lambda parameter: _Parameter(module, parameter, interface), + parsed_method.response_parameter_list)) + method.attributes = _AttributeListToDict(module, method, + parsed_method.attribute_list) + + # Enforce that only methods with response can have a [Sync] attribute. + if method.sync and method.response_parameters is None: + raise Exception("Only methods with response can include a [Sync] " + "attribute. If no response parameters are needed, you " + "could use an empty response parameter list, i.e., " + "\"=> ()\".") + # And only methods with the [Sync] attribute can specify [NoInterrupt]. + if not method.allow_interrupt and not method.sync: + raise Exception("Only [Sync] methods can be marked [NoInterrupt].") + + return method + + +def _Interface(module, parsed_iface): + """ + Args: + module: {mojom.Module} Module currently being constructed. + parsed_iface: {ast.Interface} Parsed interface. + + Returns: + {mojom.Interface} AST interface. + """ + interface = mojom.Interface(module=module) + interface.mojom_name = parsed_iface.mojom_name + interface.spec = 'x:' + module.GetNamespacePrefix() + interface.mojom_name + module.kinds[interface.spec] = interface + interface.attributes = _AttributeListToDict(module, interface, + parsed_iface.attribute_list) + interface.enums = [] + interface.constants = [] + interface.methods_data = [] + _ProcessElements( + parsed_iface.mojom_name, parsed_iface.body, { + ast.Enum: + lambda enum: interface.enums.append(_Enum(module, enum, interface)), + ast.Const: + lambda const: interface.constants.append( + _Constant(module, const, interface)), + ast.Method: + interface.methods_data.append, + }) + return interface + + +def _EnumField(module, enum, parsed_field): + """ + Args: + module: {mojom.Module} Module currently being constructed. + enum: {mojom.Enum} Enum this field belongs to. + parsed_field: {ast.EnumValue} Parsed enum value. + + Returns: + {mojom.EnumField} AST enum field. + """ + field = mojom.EnumField() + field.mojom_name = parsed_field.mojom_name + field.value = _LookupValue(module, enum, None, parsed_field.value) + field.attributes = _AttributeListToDict(module, field, + parsed_field.attribute_list) + value = mojom.EnumValue(module, enum, field) + module.values[value.GetSpec()] = value + return field + + +def _ResolveNumericEnumValues(enum): + """ + Given a reference to a mojom.Enum, resolves and assigns the numeric value of + each field, and also computes the min_value and max_value of the enum. + """ + + # map of -> integral value + prev_value = -1 + min_value = None + max_value = None + for field in enum.fields: + # This enum value is +1 the previous enum value (e.g: BEGIN). + if field.value is None: + prev_value += 1 + + # Integral value (e.g: BEGIN = -0x1). + elif isinstance(field.value, str): + prev_value = int(field.value, 0) + + # Reference to a previous enum value (e.g: INIT = BEGIN). + elif isinstance(field.value, mojom.EnumValue): + prev_value = field.value.field.numeric_value + elif isinstance(field.value, mojom.ConstantValue): + constant = field.value.constant + kind = constant.kind + if not mojom.IsIntegralKind(kind) or mojom.IsBoolKind(kind): + raise ValueError('Enum values must be integers. %s is not an integer.' % + constant.mojom_name) + prev_value = int(constant.value, 0) + else: + raise Exception('Unresolved enum value for %s' % field.value.GetSpec()) + + if prev_value in (-128, -127): + raise Exception(f'{field.mojom_name} in {enum.spec} has the value ' + f'{prev_value}, which is reserved for WTF::HashTrait\'s ' + 'default enum specialization and may not be used.') + field.numeric_value = prev_value + if min_value is None or prev_value < min_value: + min_value = prev_value + if max_value is None or prev_value > max_value: + max_value = prev_value + + enum.min_value = min_value + enum.max_value = max_value + + +def _Enum(module, parsed_enum, parent_kind): + """ + Args: + module: {mojom.Module} Module currently being constructed. + parsed_enum: {ast.Enum} Parsed enum. + + Returns: + {mojom.Enum} AST enum. + """ + enum = mojom.Enum(module=module) + enum.mojom_name = parsed_enum.mojom_name + enum.native_only = parsed_enum.enum_value_list is None + mojom_name = enum.mojom_name + if parent_kind: + mojom_name = parent_kind.mojom_name + '.' + mojom_name + enum.spec = 'x:%s.%s' % (module.mojom_namespace, mojom_name) + enum.parent_kind = parent_kind + enum.attributes = _AttributeListToDict(module, enum, + parsed_enum.attribute_list) + + if not enum.native_only: + enum.fields = list( + map(lambda field: _EnumField(module, enum, field), + parsed_enum.enum_value_list)) + _ResolveNumericEnumValues(enum) + # TODO(https://crbug.com/731893): Require a default value to be + # specified. + for field in enum.fields: + if field.default: + if not enum.extensible: + raise Exception( + f'Non-extensible enum {enum.spec} may not specify a default') + if enum.default_field is not None: + raise Exception(f'Multiple [Default] enumerators in enum {enum.spec}') + enum.default_field = field + # While running the backwards compatibility check, ignore errors because the + # old version of the enum might not specify [Default]. + if (enum.extensible and enum.default_field is None + and enum.spec not in _EXTENSIBLE_ENUMS_MISSING_DEFAULT + and not is_running_backwards_compatibility_check_hack): + raise Exception( + f'Extensible enum {enum.spec} must specify a [Default] enumerator') + + module.kinds[enum.spec] = enum + + # Enforce that a [Native] attribute is set to make native-only enum + # declarations more explicit. + if enum.native_only: + if not enum.attributes or not enum.attributes.get('Native', False): + raise Exception("Native-only enum declarations must include a " + + "Native attribute.") + + return enum + + +def _Constant(module, parsed_const, parent_kind): + """ + Args: + module: {mojom.Module} Module currently being constructed. + parsed_const: {ast.Const} Parsed constant. + + Returns: + {mojom.Constant} AST constant. + """ + constant = mojom.Constant() + constant.mojom_name = parsed_const.mojom_name + if parent_kind: + scope = (module.mojom_namespace, parent_kind.mojom_name) + else: + scope = (module.mojom_namespace, ) + # TODO(mpcomplete): maybe we should only support POD kinds. + constant.kind = _Kind(module.kinds, _MapKind(parsed_const.typename), scope) + constant.parent_kind = parent_kind + constant.value = _LookupValue(module, parent_kind, constant.kind, + parsed_const.value) + + # Iteratively resolve this constant reference to a concrete value + while isinstance(constant.value, mojom.ConstantValue): + constant.value = constant.value.constant.value + + value = mojom.ConstantValue(module, parent_kind, constant) + module.values[value.GetSpec()] = value + return constant + + +def _CollectReferencedKinds(module, all_defined_kinds): + """ + Takes a {mojom.Module} object and a list of all defined kinds within that + module, and enumerates the complete dict of user-defined mojom types + (as {mojom.Kind} objects) referenced by the module's own defined kinds (i.e. + as types of struct or union or interface parameters. The returned dict is + keyed by kind spec. + """ + + def extract_referenced_user_kinds(kind): + if mojom.IsArrayKind(kind): + return extract_referenced_user_kinds(kind.kind) + if mojom.IsMapKind(kind): + return (extract_referenced_user_kinds(kind.key_kind) + + extract_referenced_user_kinds(kind.value_kind)) + if (mojom.IsInterfaceRequestKind(kind) or mojom.IsAssociatedKind(kind) + or mojom.IsPendingRemoteKind(kind) + or mojom.IsPendingReceiverKind(kind)): + return [kind.kind] + if mojom.IsStructKind(kind): + return [kind] + if (mojom.IsInterfaceKind(kind) or mojom.IsEnumKind(kind) + or mojom.IsUnionKind(kind)): + return [kind] + return [] + + def sanitize_kind(kind): + """Removes nullability from a kind""" + if kind.spec.startswith('?'): + return _Kind(module.kinds, kind.spec[1:], (module.mojom_namespace, '')) + return kind + + referenced_user_kinds = {} + for defined_kind in all_defined_kinds: + if mojom.IsStructKind(defined_kind) or mojom.IsUnionKind(defined_kind): + for field in defined_kind.fields: + for referenced_kind in extract_referenced_user_kinds(field.kind): + sanitized_kind = sanitize_kind(referenced_kind) + referenced_user_kinds[sanitized_kind.spec] = sanitized_kind + + # Also scan for references in parameter lists + for interface in module.interfaces: + for method in interface.methods: + for param in itertools.chain(method.parameters or [], + method.response_parameters or []): + for referenced_kind in extract_referenced_user_kinds(param.kind): + sanitized_kind = sanitize_kind(referenced_kind) + referenced_user_kinds[sanitized_kind.spec] = sanitized_kind + # Consts can reference imported enums. + for const in module.constants: + if not const.kind in mojom.PRIMITIVES: + sanitized_kind = sanitize_kind(const.kind) + referenced_user_kinds[sanitized_kind.spec] = sanitized_kind + + return referenced_user_kinds + + +def _AssignDefaultOrdinals(items): + """Assigns default ordinal values to a sequence of items if necessary.""" + next_ordinal = 0 + for item in items: + if item.ordinal is not None: + next_ordinal = item.ordinal + 1 + else: + item.ordinal = next_ordinal + next_ordinal += 1 + + +def _AssertTypeIsStable(kind): + """Raises an error if a type is not stable, meaning it is composed of at least + one type that is not marked [Stable].""" + + def assertDependencyIsStable(dependency): + if (mojom.IsEnumKind(dependency) or mojom.IsStructKind(dependency) + or mojom.IsUnionKind(dependency) or mojom.IsInterfaceKind(dependency)): + if not dependency.stable: + raise Exception( + '%s is marked [Stable] but cannot be stable because it depends on ' + '%s, which is not marked [Stable].' % + (kind.mojom_name, dependency.mojom_name)) + elif mojom.IsArrayKind(dependency) or mojom.IsAnyInterfaceKind(dependency): + assertDependencyIsStable(dependency.kind) + elif mojom.IsMapKind(dependency): + assertDependencyIsStable(dependency.key_kind) + assertDependencyIsStable(dependency.value_kind) + + if mojom.IsStructKind(kind) or mojom.IsUnionKind(kind): + for field in kind.fields: + assertDependencyIsStable(field.kind) + elif mojom.IsInterfaceKind(kind): + for method in kind.methods: + for param in method.param_struct.fields: + assertDependencyIsStable(param.kind) + if method.response_param_struct: + for response_param in method.response_param_struct.fields: + assertDependencyIsStable(response_param.kind) + + +def _AssertStructIsValid(kind): + expected_ordinals = set(range(0, len(kind.fields))) + ordinals = set(map(lambda field: field.ordinal, kind.fields)) + if ordinals != expected_ordinals: + raise Exception( + 'Structs must use contiguous ordinals starting from 0. ' + + '{} is missing the following ordinals: {}.'.format( + kind.mojom_name, ', '.join(map(str, expected_ordinals - ordinals)))) + + +def _Module(tree, path, imports): + """ + Args: + tree: {ast.Mojom} The parse tree. + path: {str} The path to the mojom file. + imports: {Dict[str, mojom.Module]} Mapping from filenames, as they appear in + the import list, to already processed modules. Used to process imports. + + Returns: + {mojom.Module} An AST for the mojom. + """ + module = mojom.Module(path=path) + module.kinds = {} + for kind in mojom.PRIMITIVES: + module.kinds[kind.spec] = kind + + module.values = {} + + module.mojom_namespace = tree.module.mojom_namespace[1] if tree.module else '' + # Imports must come first, because they add to module.kinds which is used + # by by the others. + module.imports = [ + _Import(module, imports[imp.import_filename]) for imp in tree.import_list + ] + if tree.module and tree.module.attribute_list: + assert isinstance(tree.module.attribute_list, ast.AttributeList) + # TODO(vtl): Check for duplicate keys here. + module.attributes = dict((attribute.key, attribute.value) + for attribute in tree.module.attribute_list) + + filename = os.path.basename(path) + # First pass collects kinds. + module.constants = [] + module.enums = [] + module.structs = [] + module.unions = [] + module.interfaces = [] + module.features = [] + + _ProcessElements( + filename, tree.definition_list, { + ast.Const: + lambda const: module.constants.append(_Constant(module, const, None)), + ast.Enum: + lambda enum: module.enums.append(_Enum(module, enum, None)), + ast.Struct: + lambda struct: module.structs.append(_Struct(module, struct)), + ast.Union: + lambda union: module.unions.append(_Union(module, union)), + ast.Interface: + lambda interface: module.interfaces.append( + _Interface(module, interface)), + ast.Feature: + lambda feature: module.features.append(_Feature(module, feature)), + }) + + # Second pass expands fields and methods. This allows fields and parameters + # to refer to kinds defined anywhere in the mojom. + all_defined_kinds = {} + for struct in module.structs: + struct.fields = list( + map(lambda field: _StructField(module, field, struct), + struct.fields_data)) + _AssignDefaultOrdinals(struct.fields) + del struct.fields_data + all_defined_kinds[struct.spec] = struct + for enum in struct.enums: + all_defined_kinds[enum.spec] = enum + + for feature in module.features: + all_defined_kinds[feature.spec] = feature + + for union in module.unions: + union.fields = list( + map(lambda field: _UnionField(module, field, union), union.fields_data)) + _AssignDefaultOrdinals(union.fields) + for field in union.fields: + if field.is_default: + if union.default_field is not None: + raise Exception('Multiple [Default] fields in union %s.' % + union.mojom_name) + union.default_field = field + del union.fields_data + all_defined_kinds[union.spec] = union + if union.extensible and union.default_field is None: + raise Exception('Extensible union %s must specify a [Default] field' % + union.mojom_name) + + for interface in module.interfaces: + interface.methods = list( + map(lambda method: _Method(module, method, interface), + interface.methods_data)) + _AssignDefaultOrdinals(interface.methods) + del interface.methods_data + all_defined_kinds[interface.spec] = interface + for enum in interface.enums: + all_defined_kinds[enum.spec] = enum + for enum in module.enums: + all_defined_kinds[enum.spec] = enum + + all_referenced_kinds = _CollectReferencedKinds(module, + all_defined_kinds.values()) + imported_kind_specs = set(all_referenced_kinds.keys()).difference( + set(all_defined_kinds.keys())) + module.imported_kinds = OrderedDict((spec, all_referenced_kinds[spec]) + for spec in sorted(imported_kind_specs)) + + generator.AddComputedData(module) + for iface in module.interfaces: + for method in iface.methods: + if method.param_struct: + _AssignDefaultOrdinals(method.param_struct.fields) + if method.response_param_struct: + _AssignDefaultOrdinals(method.response_param_struct.fields) + + # Ensure that all types marked [Stable] are actually stable. Enums are + # automatically OK since they don't depend on other definitions. + for kinds in (module.structs, module.unions, module.interfaces): + for kind in kinds: + if kind.stable: + _AssertTypeIsStable(kind) + + for kind in module.structs: + _AssertStructIsValid(kind) + + return module + + +def OrderedModule(tree, path, imports): + """Convert parse tree to AST module. + + Args: + tree: {ast.Mojom} The parse tree. + path: {str} The path to the mojom file. + imports: {Dict[str, mojom.Module]} Mapping from filenames, as they appear in + the import list, to already processed modules. Used to process imports. + + Returns: + {mojom.Module} An AST for the mojom. + """ + module = _Module(tree, path, imports) + return module diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/translate_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/translate_unittest.py new file mode 100644 index 00000000..b4fea924 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/generate/translate_unittest.py @@ -0,0 +1,141 @@ +# Copyright 2014 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import unittest + +from mojom.generate import module as mojom +from mojom.generate import translate +from mojom.parse import ast + +class TranslateTest(unittest.TestCase): + """Tests |parser.Parse()|.""" + + def testSimpleArray(self): + """Tests a simple int32[].""" + # pylint: disable=W0212 + self.assertEquals(translate._MapKind("int32[]"), "a:i32") + + def testAssociativeArray(self): + """Tests a simple uint8{string}.""" + # pylint: disable=W0212 + self.assertEquals(translate._MapKind("uint8{string}"), "m[s][u8]") + + def testLeftToRightAssociativeArray(self): + """Makes sure that parsing is done from right to left on the internal kinds + in the presence of an associative array.""" + # pylint: disable=W0212 + self.assertEquals(translate._MapKind("uint8[]{string}"), "m[s][a:u8]") + + def testTranslateSimpleUnions(self): + """Makes sure that a simple union is translated correctly.""" + tree = ast.Mojom(None, ast.ImportList(), [ + ast.Union( + "SomeUnion", None, + ast.UnionBody([ + ast.UnionField("a", None, None, "int32"), + ast.UnionField("b", None, None, "string") + ])) + ]) + + translation = translate.OrderedModule(tree, "mojom_tree", []) + self.assertEqual(1, len(translation.unions)) + + union = translation.unions[0] + self.assertTrue(isinstance(union, mojom.Union)) + self.assertEqual("SomeUnion", union.mojom_name) + self.assertEqual(2, len(union.fields)) + self.assertEqual("a", union.fields[0].mojom_name) + self.assertEqual(mojom.INT32.spec, union.fields[0].kind.spec) + self.assertEqual("b", union.fields[1].mojom_name) + self.assertEqual(mojom.STRING.spec, union.fields[1].kind.spec) + + def testMapKindRaisesWithDuplicate(self): + """Verifies _MapTreeForType() raises when passed two values with the same + name.""" + methods = [ + ast.Method('dup', None, None, ast.ParameterList(), None), + ast.Method('dup', None, None, ast.ParameterList(), None) + ] + with self.assertRaises(Exception): + translate._ElemsOfType(methods, ast.Method, 'scope') + + def testAssociatedKinds(self): + """Tests type spec translation of associated interfaces and requests.""" + # pylint: disable=W0212 + self.assertEquals( + translate._MapKind("asso?"), "?asso:x:SomeInterface") + self.assertEquals(translate._MapKind("rca?"), + "?rca:x:SomeInterface") + + def testSelfRecursiveUnions(self): + """Verifies _UnionField() raises when a union is self-recursive.""" + tree = ast.Mojom(None, ast.ImportList(), [ + ast.Union("SomeUnion", None, + ast.UnionBody([ast.UnionField("a", None, None, "SomeUnion")])) + ]) + with self.assertRaises(Exception): + translate.OrderedModule(tree, "mojom_tree", []) + + tree = ast.Mojom(None, ast.ImportList(), [ + ast.Union( + "SomeUnion", None, + ast.UnionBody([ast.UnionField("a", None, None, "SomeUnion?")])) + ]) + with self.assertRaises(Exception): + translate.OrderedModule(tree, "mojom_tree", []) + + def testDuplicateAttributesException(self): + tree = ast.Mojom(None, ast.ImportList(), [ + ast.Union( + "FakeUnion", + ast.AttributeList([ + ast.Attribute("key1", "value"), + ast.Attribute("key1", "value") + ]), + ast.UnionBody([ + ast.UnionField("a", None, None, "int32"), + ast.UnionField("b", None, None, "string") + ])) + ]) + with self.assertRaises(Exception): + translate.OrderedModule(tree, "mojom_tree", []) + + def testEnumWithReservedValues(self): + """Verifies that assigning reserved values to enumerators fails.""" + # -128 is reserved for the empty representation in WTF::HashTraits. + tree = ast.Mojom(None, ast.ImportList(), [ + ast.Enum( + "MyEnum", None, + ast.EnumValueList([ + ast.EnumValue('kReserved', None, '-128'), + ])) + ]) + with self.assertRaises(Exception) as context: + translate.OrderedModule(tree, "mojom_tree", []) + self.assertIn("reserved for WTF::HashTrait", str(context.exception)) + + # -127 is reserved for the deleted representation in WTF::HashTraits. + tree = ast.Mojom(None, ast.ImportList(), [ + ast.Enum( + "MyEnum", None, + ast.EnumValueList([ + ast.EnumValue('kReserved', None, '-127'), + ])) + ]) + with self.assertRaises(Exception) as context: + translate.OrderedModule(tree, "mojom_tree", []) + self.assertIn("reserved for WTF::HashTrait", str(context.exception)) + + # Implicitly assigning a reserved value should also fail. + tree = ast.Mojom(None, ast.ImportList(), [ + ast.Enum( + "MyEnum", None, + ast.EnumValueList([ + ast.EnumValue('kNotReserved', None, '-129'), + ast.EnumValue('kImplicitlyReserved', None, None), + ])) + ]) + with self.assertRaises(Exception) as context: + translate.OrderedModule(tree, "mojom_tree", []) + self.assertIn("reserved for WTF::HashTrait", str(context.exception)) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/__init__.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/ast.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/ast.py new file mode 100644 index 00000000..aae9cdb6 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/ast.py @@ -0,0 +1,462 @@ +# Copyright 2014 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Node classes for the AST for a Mojo IDL file.""" + +# Note: For convenience of testing, you probably want to define __eq__() methods +# for all node types; it's okay to be slightly lax (e.g., not compare filename +# and lineno). You may also define __repr__() to help with analyzing test +# failures, especially for more complex types. + +import os.path + + +# Instance of 'NodeListBase' has no '_list_item_type' member (no-member) +# pylint: disable=no-member + + +class NodeBase: + """Base class for nodes in the AST.""" + + def __init__(self, filename=None, lineno=None): + self.filename = filename + self.lineno = lineno + + def __eq__(self, other): + # We want strict comparison of the two object's types. Disable pylint's + # insistence upon recommending isinstance(). + # pylint: disable=unidiomatic-typecheck + return type(self) == type(other) + + # Make != the inverse of ==. (Subclasses shouldn't have to override this.) + def __ne__(self, other): + return not self == other + + +# TODO(vtl): Some of this is complicated enough that it should be tested. +class NodeListBase(NodeBase): + """Represents a list of other nodes, all having the same type. (This is meant + to be subclassed, with subclasses defining _list_item_type to be the class (or + classes, in a tuple) of the members of the list.)""" + + def __init__(self, item_or_items=None, **kwargs): + super().__init__(**kwargs) + self.items = [] + if item_or_items is None: + pass + elif isinstance(item_or_items, list): + for item in item_or_items: + assert isinstance(item, self._list_item_type) + self.Append(item) + else: + assert isinstance(item_or_items, self._list_item_type) + self.Append(item_or_items) + + # Support iteration. For everything else, users should just access |items| + # directly. (We intentionally do NOT supply |__len__()| or |__nonzero__()|, so + # |bool(NodeListBase())| is true.) + def __iter__(self): + return self.items.__iter__() + + def __eq__(self, other): + return super().__eq__(other) and \ + self.items == other.items + + # Implement this so that on failure, we get slightly more sensible output. + def __repr__(self): + return self.__class__.__name__ + "([" + \ + ", ".join([repr(elem) for elem in self.items]) + "])" + + def Insert(self, item): + """Inserts item at the front of the list.""" + + assert isinstance(item, self._list_item_type) + self.items.insert(0, item) + self._UpdateFilenameAndLineno() + + def Append(self, item): + """Appends item to the end of the list.""" + + assert isinstance(item, self._list_item_type) + self.items.append(item) + self._UpdateFilenameAndLineno() + + def _UpdateFilenameAndLineno(self): + if self.items: + self.filename = self.items[0].filename + self.lineno = self.items[0].lineno + + +class Definition(NodeBase): + """Represents a definition of anything that has a global name (e.g., enums, + enum values, consts, structs, struct fields, interfaces). (This does not + include parameter definitions.) This class is meant to be subclassed.""" + + def __init__(self, mojom_name, **kwargs): + assert isinstance(mojom_name, str) + NodeBase.__init__(self, **kwargs) + self.mojom_name = mojom_name + + +################################################################################ + + +class Attribute(NodeBase): + """Represents an attribute.""" + + def __init__(self, key, value, **kwargs): + assert isinstance(key, str) + super().__init__(**kwargs) + self.key = key + self.value = value + + def __eq__(self, other): + return super().__eq__(other) and \ + self.key == other.key and \ + self.value == other.value + + +class AttributeList(NodeListBase): + """Represents a list attributes.""" + + _list_item_type = Attribute + + +class Const(Definition): + """Represents a const definition.""" + + def __init__(self, mojom_name, attribute_list, typename, value, **kwargs): + assert attribute_list is None or isinstance(attribute_list, AttributeList) + # The typename is currently passed through as a string. + assert isinstance(typename, str) + # The value is either a literal (currently passed through as a string) or a + # "wrapped identifier". + assert isinstance(value, (tuple, str)) + super().__init__(mojom_name, **kwargs) + self.attribute_list = attribute_list + self.typename = typename + self.value = value + + def __eq__(self, other): + return super().__eq__(other) and \ + self.attribute_list == other.attribute_list and \ + self.typename == other.typename and \ + self.value == other.value + + +class Enum(Definition): + """Represents an enum definition.""" + + def __init__(self, mojom_name, attribute_list, enum_value_list, **kwargs): + assert attribute_list is None or isinstance(attribute_list, AttributeList) + assert enum_value_list is None or isinstance(enum_value_list, EnumValueList) + super().__init__(mojom_name, **kwargs) + self.attribute_list = attribute_list + self.enum_value_list = enum_value_list + + def __eq__(self, other): + return super().__eq__(other) and \ + self.attribute_list == other.attribute_list and \ + self.enum_value_list == other.enum_value_list + + +class EnumValue(Definition): + """Represents a definition of an enum value.""" + + def __init__(self, mojom_name, attribute_list, value, **kwargs): + # The optional value is either an int (which is current a string) or a + # "wrapped identifier". + assert attribute_list is None or isinstance(attribute_list, AttributeList) + assert value is None or isinstance(value, (tuple, str)) + super().__init__(mojom_name, **kwargs) + self.attribute_list = attribute_list + self.value = value + + def __eq__(self, other): + return super().__eq__(other) and \ + self.attribute_list == other.attribute_list and \ + self.value == other.value + + +class EnumValueList(NodeListBase): + """Represents a list of enum value definitions (i.e., the "body" of an enum + definition).""" + + _list_item_type = EnumValue + + +class Feature(Definition): + """Represents a runtime feature definition.""" + def __init__(self, mojom_name, attribute_list, body, **kwargs): + assert attribute_list is None or isinstance(attribute_list, AttributeList) + assert isinstance(body, FeatureBody) or body is None + super().__init__(mojom_name, **kwargs) + self.attribute_list = attribute_list + self.body = body + + def __eq__(self, other): + return super().__eq__(other) and \ + self.attribute_list == other.attribute_list and \ + self.body == other.body + + def __repr__(self): + return "Feature(mojom_name = %s, attribute_list = %s, body = %s)" % ( + self.mojom_name, self.attribute_list, self.body) + + +# This needs to be declared after `FeatureConst` and `FeatureField`. +class FeatureBody(NodeListBase): + """Represents the body of (i.e., list of definitions inside) a feature.""" + + # Features are compile time helpers so all fields are initializers/consts + # for the underlying platform feature type. + _list_item_type = (Const) + + +class Import(NodeBase): + """Represents an import statement.""" + + def __init__(self, attribute_list, import_filename, **kwargs): + assert attribute_list is None or isinstance(attribute_list, AttributeList) + assert isinstance(import_filename, str) + super().__init__(**kwargs) + self.attribute_list = attribute_list + # TODO(crbug.com/953884): Use pathlib once we're migrated fully to Python 3. + self.import_filename = os.path.normpath(import_filename).replace('\\', '/') + + def __eq__(self, other): + return super().__eq__(other) and \ + self.attribute_list == other.attribute_list and \ + self.import_filename == other.import_filename + + +class ImportList(NodeListBase): + """Represents a list (i.e., sequence) of import statements.""" + + _list_item_type = Import + + +class Interface(Definition): + """Represents an interface definition.""" + + def __init__(self, mojom_name, attribute_list, body, **kwargs): + assert attribute_list is None or isinstance(attribute_list, AttributeList) + assert isinstance(body, InterfaceBody) + super().__init__(mojom_name, **kwargs) + self.attribute_list = attribute_list + self.body = body + + def __eq__(self, other): + return super().__eq__(other) and \ + self.attribute_list == other.attribute_list and \ + self.body == other.body + + +class Method(Definition): + """Represents a method definition.""" + + def __init__(self, mojom_name, attribute_list, ordinal, parameter_list, + response_parameter_list, **kwargs): + assert attribute_list is None or isinstance(attribute_list, AttributeList) + assert ordinal is None or isinstance(ordinal, Ordinal) + assert isinstance(parameter_list, ParameterList) + assert response_parameter_list is None or \ + isinstance(response_parameter_list, ParameterList) + super().__init__(mojom_name, **kwargs) + self.attribute_list = attribute_list + self.ordinal = ordinal + self.parameter_list = parameter_list + self.response_parameter_list = response_parameter_list + + def __eq__(self, other): + return super().__eq__(other) and \ + self.attribute_list == other.attribute_list and \ + self.ordinal == other.ordinal and \ + self.parameter_list == other.parameter_list and \ + self.response_parameter_list == other.response_parameter_list + + +# This needs to be declared after |Method|. +class InterfaceBody(NodeListBase): + """Represents the body of (i.e., list of definitions inside) an interface.""" + + _list_item_type = (Const, Enum, Method) + + +class Module(NodeBase): + """Represents a module statement.""" + + def __init__(self, mojom_namespace, attribute_list, **kwargs): + # |mojom_namespace| is either none or a "wrapped identifier". + assert mojom_namespace is None or isinstance(mojom_namespace, tuple) + assert attribute_list is None or isinstance(attribute_list, AttributeList) + super().__init__(**kwargs) + self.mojom_namespace = mojom_namespace + self.attribute_list = attribute_list + + def __eq__(self, other): + return super().__eq__(other) and \ + self.mojom_namespace == other.mojom_namespace and \ + self.attribute_list == other.attribute_list + + +class Mojom(NodeBase): + """Represents an entire .mojom file. (This is the root node.)""" + + def __init__(self, module, import_list, definition_list, **kwargs): + assert module is None or isinstance(module, Module) + assert isinstance(import_list, ImportList) + assert isinstance(definition_list, list) + super().__init__(**kwargs) + self.module = module + self.import_list = import_list + self.definition_list = definition_list + + def __eq__(self, other): + return super().__eq__(other) and \ + self.module == other.module and \ + self.import_list == other.import_list and \ + self.definition_list == other.definition_list + + def __repr__(self): + return "%s(%r, %r, %r)" % (self.__class__.__name__, self.module, + self.import_list, self.definition_list) + + +class Ordinal(NodeBase): + """Represents an ordinal value labeling, e.g., a struct field.""" + + def __init__(self, value, **kwargs): + assert isinstance(value, int) + super().__init__(**kwargs) + self.value = value + + def __eq__(self, other): + return super().__eq__(other) and \ + self.value == other.value + + +class Parameter(NodeBase): + """Represents a method request or response parameter.""" + + def __init__(self, mojom_name, attribute_list, ordinal, typename, **kwargs): + assert isinstance(mojom_name, str) + assert attribute_list is None or isinstance(attribute_list, AttributeList) + assert ordinal is None or isinstance(ordinal, Ordinal) + assert isinstance(typename, str) + super().__init__(**kwargs) + self.mojom_name = mojom_name + self.attribute_list = attribute_list + self.ordinal = ordinal + self.typename = typename + + def __eq__(self, other): + return super().__eq__(other) and \ + self.mojom_name == other.mojom_name and \ + self.attribute_list == other.attribute_list and \ + self.ordinal == other.ordinal and \ + self.typename == other.typename + + +class ParameterList(NodeListBase): + """Represents a list of (method request or response) parameters.""" + + _list_item_type = Parameter + + +class Struct(Definition): + """Represents a struct definition.""" + + def __init__(self, mojom_name, attribute_list, body, **kwargs): + assert attribute_list is None or isinstance(attribute_list, AttributeList) + assert isinstance(body, StructBody) or body is None + super().__init__(mojom_name, **kwargs) + self.attribute_list = attribute_list + self.body = body + + def __eq__(self, other): + return super().__eq__(other) and \ + self.attribute_list == other.attribute_list and \ + self.body == other.body + + def __repr__(self): + return "Struct(mojom_name = %s, attribute_list = %s, body = %s)" % ( + self.mojom_name, self.attribute_list, self.body) + + +class StructField(Definition): + """Represents a struct field definition.""" + + def __init__(self, mojom_name, attribute_list, ordinal, typename, + default_value, **kwargs): + assert isinstance(mojom_name, str) + assert attribute_list is None or isinstance(attribute_list, AttributeList) + assert ordinal is None or isinstance(ordinal, Ordinal) + assert isinstance(typename, str) + # The optional default value is currently either a value as a string or a + # "wrapped identifier". + assert default_value is None or isinstance(default_value, (str, tuple)) + super().__init__(mojom_name, **kwargs) + self.attribute_list = attribute_list + self.ordinal = ordinal + self.typename = typename + self.default_value = default_value + + def __eq__(self, other): + return super().__eq__(other) and \ + self.attribute_list == other.attribute_list and \ + self.ordinal == other.ordinal and \ + self.typename == other.typename and \ + self.default_value == other.default_value + + def __repr__(self): + return ("StructField(mojom_name = %s, attribute_list = %s, ordinal = %s, " + "typename = %s, default_value = %s") % ( + self.mojom_name, self.attribute_list, self.ordinal, + self.typename, self.default_value) + + +# This needs to be declared after |StructField|. +class StructBody(NodeListBase): + """Represents the body of (i.e., list of definitions inside) a struct.""" + + _list_item_type = (Const, Enum, StructField) + + +class Union(Definition): + """Represents a union definition.""" + + def __init__(self, mojom_name, attribute_list, body, **kwargs): + assert attribute_list is None or isinstance(attribute_list, AttributeList) + assert isinstance(body, UnionBody) + super().__init__(mojom_name, **kwargs) + self.attribute_list = attribute_list + self.body = body + + def __eq__(self, other): + return super().__eq__(other) and \ + self.attribute_list == other.attribute_list and \ + self.body == other.body + + +class UnionField(Definition): + def __init__(self, mojom_name, attribute_list, ordinal, typename, **kwargs): + assert isinstance(mojom_name, str) + assert attribute_list is None or isinstance(attribute_list, AttributeList) + assert ordinal is None or isinstance(ordinal, Ordinal) + assert isinstance(typename, str) + super().__init__(mojom_name, **kwargs) + self.attribute_list = attribute_list + self.ordinal = ordinal + self.typename = typename + + def __eq__(self, other): + return super().__eq__(other) and \ + self.attribute_list == other.attribute_list and \ + self.ordinal == other.ordinal and \ + self.typename == other.typename + + +class UnionBody(NodeListBase): + + _list_item_type = UnionField diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/ast_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/ast_unittest.py new file mode 100644 index 00000000..b289f7b1 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/ast_unittest.py @@ -0,0 +1,115 @@ +# Copyright 2014 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import unittest + +from mojom.parse import ast + +class _TestNode(ast.NodeBase): + """Node type for tests.""" + + def __init__(self, value, **kwargs): + super().__init__(**kwargs) + self.value = value + + def __eq__(self, other): + return super().__eq__(other) and self.value == other.value + +class _TestNodeList(ast.NodeListBase): + """Node list type for tests.""" + + _list_item_type = _TestNode + +class ASTTest(unittest.TestCase): + """Tests various AST classes.""" + + def testNodeBase(self): + # Test |__eq__()|; this is only used for testing, where we want to do + # comparison by value and ignore filenames/line numbers (for convenience). + node1 = ast.NodeBase(filename="hello.mojom", lineno=123) + node2 = ast.NodeBase() + self.assertEquals(node1, node2) + self.assertEquals(node2, node1) + + # Check that |__ne__()| just defers to |__eq__()| properly. + self.assertFalse(node1 != node2) + self.assertFalse(node2 != node1) + + # Check that |filename| and |lineno| are set properly (and are None by + # default). + self.assertEquals(node1.filename, "hello.mojom") + self.assertEquals(node1.lineno, 123) + self.assertIsNone(node2.filename) + self.assertIsNone(node2.lineno) + + # |NodeBase|'s |__eq__()| should compare types (and a subclass's |__eq__()| + # should first defer to its superclass's). + node3 = _TestNode(123) + self.assertNotEqual(node1, node3) + self.assertNotEqual(node3, node1) + # Also test |__eq__()| directly. + self.assertFalse(node1 == node3) + self.assertFalse(node3 == node1) + + node4 = _TestNode(123, filename="world.mojom", lineno=123) + self.assertEquals(node4, node3) + node5 = _TestNode(456) + self.assertNotEquals(node5, node4) + + def testNodeListBase(self): + node1 = _TestNode(1, filename="foo.mojom", lineno=1) + # Equal to, but not the same as, |node1|: + node1b = _TestNode(1, filename="foo.mojom", lineno=1) + node2 = _TestNode(2, filename="foo.mojom", lineno=2) + + nodelist1 = _TestNodeList() # Contains: (empty). + self.assertEquals(nodelist1, nodelist1) + self.assertEquals(nodelist1.items, []) + self.assertIsNone(nodelist1.filename) + self.assertIsNone(nodelist1.lineno) + + nodelist2 = _TestNodeList(node1) # Contains: 1. + self.assertEquals(nodelist2, nodelist2) + self.assertEquals(nodelist2.items, [node1]) + self.assertNotEqual(nodelist2, nodelist1) + self.assertEquals(nodelist2.filename, "foo.mojom") + self.assertEquals(nodelist2.lineno, 1) + + nodelist3 = _TestNodeList([node2]) # Contains: 2. + self.assertEquals(nodelist3.items, [node2]) + self.assertNotEqual(nodelist3, nodelist1) + self.assertNotEqual(nodelist3, nodelist2) + self.assertEquals(nodelist3.filename, "foo.mojom") + self.assertEquals(nodelist3.lineno, 2) + + nodelist1.Append(node1b) # Contains: 1. + self.assertEquals(nodelist1.items, [node1]) + self.assertEquals(nodelist1, nodelist2) + self.assertNotEqual(nodelist1, nodelist3) + self.assertEquals(nodelist1.filename, "foo.mojom") + self.assertEquals(nodelist1.lineno, 1) + + nodelist1.Append(node2) # Contains: 1, 2. + self.assertEquals(nodelist1.items, [node1, node2]) + self.assertNotEqual(nodelist1, nodelist2) + self.assertNotEqual(nodelist1, nodelist3) + self.assertEquals(nodelist1.lineno, 1) + + nodelist2.Append(node2) # Contains: 1, 2. + self.assertEquals(nodelist2.items, [node1, node2]) + self.assertEquals(nodelist2, nodelist1) + self.assertNotEqual(nodelist2, nodelist3) + self.assertEquals(nodelist2.lineno, 1) + + nodelist3.Insert(node1) # Contains: 1, 2. + self.assertEquals(nodelist3.items, [node1, node2]) + self.assertEquals(nodelist3, nodelist1) + self.assertEquals(nodelist3, nodelist2) + self.assertEquals(nodelist3.lineno, 1) + + # Test iteration: + i = 1 + for item in nodelist1: + self.assertEquals(item.value, i) + i += 1 diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/conditional_features.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/conditional_features.py new file mode 100644 index 00000000..9687edbf --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/conditional_features.py @@ -0,0 +1,83 @@ +# Copyright 2018 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Helpers for processing conditionally enabled features in a mojom.""" + +from mojom.error import Error +from mojom.parse import ast + + +class EnableIfError(Error): + """ Class for errors from .""" + + def __init__(self, filename, message, lineno=None): + Error.__init__(self, filename, message, lineno=lineno, addenda=None) + + +def _IsEnabled(definition, enabled_features): + """Returns true if a definition is enabled. + + A definition is enabled if it has no EnableIf/EnableIfNot attribute. + It is retained if it has an EnableIf attribute and the attribute is in + enabled_features. It is retained if it has an EnableIfNot attribute and the + attribute is not in enabled features. + """ + if not hasattr(definition, "attribute_list"): + return True + if not definition.attribute_list: + return True + + already_defined = False + for a in definition.attribute_list: + if a.key == 'EnableIf' or a.key == 'EnableIfNot': + if already_defined: + raise EnableIfError( + definition.filename, + "EnableIf/EnableIfNot attribute may only be set once per field.", + definition.lineno) + already_defined = True + + for attribute in definition.attribute_list: + if attribute.key == 'EnableIf' and attribute.value not in enabled_features: + return False + if attribute.key == 'EnableIfNot' and attribute.value in enabled_features: + return False + return True + + +def _FilterDisabledFromNodeList(node_list, enabled_features): + if not node_list: + return + assert isinstance(node_list, ast.NodeListBase) + node_list.items = [ + item for item in node_list.items if _IsEnabled(item, enabled_features) + ] + for item in node_list.items: + _FilterDefinition(item, enabled_features) + + +def _FilterDefinition(definition, enabled_features): + """Filters definitions with a body.""" + if isinstance(definition, ast.Enum): + _FilterDisabledFromNodeList(definition.enum_value_list, enabled_features) + elif isinstance(definition, ast.Method): + _FilterDisabledFromNodeList(definition.parameter_list, enabled_features) + _FilterDisabledFromNodeList(definition.response_parameter_list, + enabled_features) + elif isinstance(definition, + (ast.Interface, ast.Struct, ast.Union, ast.Feature)): + _FilterDisabledFromNodeList(definition.body, enabled_features) + + +def RemoveDisabledDefinitions(mojom, enabled_features): + """Removes conditionally disabled definitions from a Mojom node.""" + mojom.import_list = ast.ImportList([ + imported_file for imported_file in mojom.import_list + if _IsEnabled(imported_file, enabled_features) + ]) + mojom.definition_list = [ + definition for definition in mojom.definition_list + if _IsEnabled(definition, enabled_features) + ] + for definition in mojom.definition_list: + _FilterDefinition(definition, enabled_features) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/conditional_features_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/conditional_features_unittest.py new file mode 100644 index 00000000..cca1764b --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/conditional_features_unittest.py @@ -0,0 +1,376 @@ +# Copyright 2018 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import importlib.util +import os +import sys +import unittest + +def _GetDirAbove(dirname): + """Returns the directory "above" this file containing |dirname| (which must + also be "above" this file).""" + path = os.path.abspath(__file__) + while True: + path, tail = os.path.split(path) + assert tail + if tail == dirname: + return path + +try: + importlib.util.find_spec("mojom") +except ImportError: + sys.path.append(os.path.join(_GetDirAbove('pylib'), 'pylib')) +import mojom.parse.ast as ast +import mojom.parse.conditional_features as conditional_features +import mojom.parse.parser as parser + +ENABLED_FEATURES = frozenset({'red', 'green', 'blue'}) + +class ConditionalFeaturesTest(unittest.TestCase): + """Tests |mojom.parse.conditional_features|.""" + + def parseAndAssertEqual(self, source, expected_source): + definition = parser.Parse(source, "my_file.mojom") + conditional_features.RemoveDisabledDefinitions(definition, ENABLED_FEATURES) + expected = parser.Parse(expected_source, "my_file.mojom") + self.assertEquals(definition, expected) + + def testFilterConst(self): + """Test that Consts are correctly filtered.""" + const_source = """ + [EnableIf=blue] + const int kMyConst1 = 1; + [EnableIf=orange] + const double kMyConst2 = 2; + const int kMyConst3 = 3; + """ + expected_source = """ + [EnableIf=blue] + const int kMyConst1 = 1; + const int kMyConst3 = 3; + """ + self.parseAndAssertEqual(const_source, expected_source) + + def testFilterIfNotConst(self): + """Test that Consts are correctly filtered.""" + const_source = """ + [EnableIfNot=blue] + const int kMyConst1 = 1; + [EnableIfNot=orange] + const double kMyConst2 = 2; + [EnableIf=blue] + const int kMyConst3 = 3; + [EnableIfNot=blue] + const int kMyConst4 = 4; + [EnableIfNot=purple] + const int kMyConst5 = 5; + """ + expected_source = """ + [EnableIfNot=orange] + const double kMyConst2 = 2; + [EnableIf=blue] + const int kMyConst3 = 3; + [EnableIfNot=purple] + const int kMyConst5 = 5; + """ + self.parseAndAssertEqual(const_source, expected_source) + + def testFilterIfNotMultipleConst(self): + """Test that Consts are correctly filtered.""" + const_source = """ + [EnableIfNot=blue] + const int kMyConst1 = 1; + [EnableIfNot=orange] + const double kMyConst2 = 2; + [EnableIfNot=orange] + const int kMyConst3 = 3; + """ + expected_source = """ + [EnableIfNot=orange] + const double kMyConst2 = 2; + [EnableIfNot=orange] + const int kMyConst3 = 3; + """ + self.parseAndAssertEqual(const_source, expected_source) + + def testFilterEnum(self): + """Test that EnumValues are correctly filtered from an Enum.""" + enum_source = """ + enum MyEnum { + [EnableIf=purple] + VALUE1, + [EnableIf=blue] + VALUE2, + VALUE3, + }; + """ + expected_source = """ + enum MyEnum { + [EnableIf=blue] + VALUE2, + VALUE3 + }; + """ + self.parseAndAssertEqual(enum_source, expected_source) + + def testFilterImport(self): + """Test that imports are correctly filtered from a Mojom.""" + import_source = """ + [EnableIf=blue] + import "foo.mojom"; + import "bar.mojom"; + [EnableIf=purple] + import "baz.mojom"; + """ + expected_source = """ + [EnableIf=blue] + import "foo.mojom"; + import "bar.mojom"; + """ + self.parseAndAssertEqual(import_source, expected_source) + + def testFilterIfNotImport(self): + """Test that imports are correctly filtered from a Mojom.""" + import_source = """ + [EnableIf=blue] + import "foo.mojom"; + [EnableIfNot=purple] + import "bar.mojom"; + [EnableIfNot=green] + import "baz.mojom"; + """ + expected_source = """ + [EnableIf=blue] + import "foo.mojom"; + [EnableIfNot=purple] + import "bar.mojom"; + """ + self.parseAndAssertEqual(import_source, expected_source) + + def testFilterInterface(self): + """Test that definitions are correctly filtered from an Interface.""" + interface_source = """ + interface MyInterface { + [EnableIf=blue] + enum MyEnum { + [EnableIf=purple] + VALUE1, + VALUE2, + }; + [EnableIf=blue] + const int32 kMyConst = 123; + [EnableIf=purple] + MyMethod(); + }; + """ + expected_source = """ + interface MyInterface { + [EnableIf=blue] + enum MyEnum { + VALUE2, + }; + [EnableIf=blue] + const int32 kMyConst = 123; + }; + """ + self.parseAndAssertEqual(interface_source, expected_source) + + def testFilterMethod(self): + """Test that Parameters are correctly filtered from a Method.""" + method_source = """ + interface MyInterface { + [EnableIf=blue] + MyMethod([EnableIf=purple] int32 a) => ([EnableIf=red] int32 b); + }; + """ + expected_source = """ + interface MyInterface { + [EnableIf=blue] + MyMethod() => ([EnableIf=red] int32 b); + }; + """ + self.parseAndAssertEqual(method_source, expected_source) + + def testFilterStruct(self): + """Test that definitions are correctly filtered from a Struct.""" + struct_source = """ + struct MyStruct { + [EnableIf=blue] + enum MyEnum { + VALUE1, + [EnableIf=purple] + VALUE2, + }; + [EnableIf=yellow] + const double kMyConst = 1.23; + [EnableIf=green] + int32 a; + double b; + [EnableIf=purple] + int32 c; + [EnableIf=blue] + double d; + int32 e; + [EnableIf=orange] + double f; + }; + """ + expected_source = """ + struct MyStruct { + [EnableIf=blue] + enum MyEnum { + VALUE1, + }; + [EnableIf=green] + int32 a; + double b; + [EnableIf=blue] + double d; + int32 e; + }; + """ + self.parseAndAssertEqual(struct_source, expected_source) + + def testFilterIfNotStruct(self): + """Test that definitions are correctly filtered from a Struct.""" + struct_source = """ + struct MyStruct { + [EnableIf=blue] + enum MyEnum { + VALUE1, + [EnableIfNot=red] + VALUE2, + }; + [EnableIfNot=yellow] + const double kMyConst = 1.23; + [EnableIf=green] + int32 a; + double b; + [EnableIfNot=purple] + int32 c; + [EnableIf=blue] + double d; + int32 e; + [EnableIfNot=red] + double f; + }; + """ + expected_source = """ + struct MyStruct { + [EnableIf=blue] + enum MyEnum { + VALUE1, + }; + [EnableIfNot=yellow] + const double kMyConst = 1.23; + [EnableIf=green] + int32 a; + double b; + [EnableIfNot=purple] + int32 c; + [EnableIf=blue] + double d; + int32 e; + }; + """ + self.parseAndAssertEqual(struct_source, expected_source) + + def testFilterUnion(self): + """Test that UnionFields are correctly filtered from a Union.""" + union_source = """ + union MyUnion { + [EnableIf=yellow] + int32 a; + [EnableIf=red] + bool b; + }; + """ + expected_source = """ + union MyUnion { + [EnableIf=red] + bool b; + }; + """ + self.parseAndAssertEqual(union_source, expected_source) + + def testSameNameFields(self): + mojom_source = """ + enum Foo { + [EnableIf=red] + VALUE1 = 5, + [EnableIf=yellow] + VALUE1 = 6, + }; + [EnableIf=red] + const double kMyConst = 1.23; + [EnableIf=yellow] + const double kMyConst = 4.56; + """ + expected_source = """ + enum Foo { + [EnableIf=red] + VALUE1 = 5, + }; + [EnableIf=red] + const double kMyConst = 1.23; + """ + self.parseAndAssertEqual(mojom_source, expected_source) + + def testFeaturesWithEnableIf(self): + mojom_source = """ + feature Foo { + const string name = "FooFeature"; + [EnableIf=red] + const bool default_state = false; + [EnableIf=yellow] + const bool default_state = true; + }; + """ + expected_source = """ + feature Foo { + const string name = "FooFeature"; + [EnableIf=red] + const bool default_state = false; + }; + """ + self.parseAndAssertEqual(mojom_source, expected_source) + + def testMultipleEnableIfs(self): + source = """ + enum Foo { + [EnableIf=red,EnableIf=yellow] + kBarValue = 5, + }; + """ + definition = parser.Parse(source, "my_file.mojom") + self.assertRaises(conditional_features.EnableIfError, + conditional_features.RemoveDisabledDefinitions, + definition, ENABLED_FEATURES) + + def testMultipleEnableIfs(self): + source = """ + enum Foo { + [EnableIf=red,EnableIfNot=yellow] + kBarValue = 5, + }; + """ + definition = parser.Parse(source, "my_file.mojom") + self.assertRaises(conditional_features.EnableIfError, + conditional_features.RemoveDisabledDefinitions, + definition, ENABLED_FEATURES) + + def testMultipleEnableIfs(self): + source = """ + enum Foo { + [EnableIfNot=red,EnableIfNot=yellow] + kBarValue = 5, + }; + """ + definition = parser.Parse(source, "my_file.mojom") + self.assertRaises(conditional_features.EnableIfError, + conditional_features.RemoveDisabledDefinitions, + definition, ENABLED_FEATURES) + +if __name__ == '__main__': + unittest.main() diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/lexer.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/lexer.py new file mode 100644 index 00000000..00136a8b --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/lexer.py @@ -0,0 +1,249 @@ +# Copyright 2014 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os.path +import sys + +from mojom import fileutil +from mojom.error import Error + +fileutil.AddLocalRepoThirdPartyDirToModulePath() +from ply.lex import TOKEN + + +class LexError(Error): + """Class for errors from the lexer.""" + + def __init__(self, filename, message, lineno): + Error.__init__(self, filename, message, lineno=lineno) + + +# We have methods which look like they could be functions: +# pylint: disable=R0201 +class Lexer: + def __init__(self, filename): + self.filename = filename + + ######################-- PRIVATE --###################### + + ## + ## Internal auxiliary methods + ## + def _error(self, msg, token): + raise LexError(self.filename, msg, token.lineno) + + ## + ## Reserved keywords + ## + keywords = ( + 'HANDLE', + 'IMPORT', + 'MODULE', + 'STRUCT', + 'UNION', + 'INTERFACE', + 'ENUM', + 'CONST', + 'TRUE', + 'FALSE', + 'DEFAULT', + 'ARRAY', + 'MAP', + 'ASSOCIATED', + 'PENDING_REMOTE', + 'PENDING_RECEIVER', + 'PENDING_ASSOCIATED_REMOTE', + 'PENDING_ASSOCIATED_RECEIVER', + 'FEATURE', + ) + + keyword_map = {} + for keyword in keywords: + keyword_map[keyword.lower()] = keyword + + ## + ## All the tokens recognized by the lexer + ## + tokens = keywords + ( + # Identifiers + 'NAME', + + # Constants + 'ORDINAL', + 'INT_CONST_DEC', + 'INT_CONST_HEX', + 'FLOAT_CONST', + + # String literals + 'STRING_LITERAL', + + # Operators + 'MINUS', + 'PLUS', + 'QSTN', + + # Assignment + 'EQUALS', + + # Request / response + 'RESPONSE', + + # Delimiters + 'LPAREN', + 'RPAREN', # ( ) + 'LBRACKET', + 'RBRACKET', # [ ] + 'LBRACE', + 'RBRACE', # { } + 'LANGLE', + 'RANGLE', # < > + 'SEMI', # ; + 'COMMA', + 'DOT' # , . + ) + + ## + ## Regexes for use in tokens + ## + + # valid C identifiers (K&R2: A.2.3) + identifier = r'[a-zA-Z_][0-9a-zA-Z_]*' + + hex_prefix = '0[xX]' + hex_digits = '[0-9a-fA-F]+' + + # integer constants (K&R2: A.2.5.1) + decimal_constant = '0|([1-9][0-9]*)' + hex_constant = hex_prefix + hex_digits + # Don't allow octal constants (even invalid octal). + octal_constant_disallowed = '0[0-9]+' + + # character constants (K&R2: A.2.5.2) + # Note: a-zA-Z and '.-~^_!=&;,' are allowed as escape chars to support #line + # directives with Windows paths as filenames (..\..\dir\file) + # For the same reason, decimal_escape allows all digit sequences. We want to + # parse all correct code, even if it means to sometimes parse incorrect + # code. + # + simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])""" + decimal_escape = r"""(\d+)""" + hex_escape = r"""(x[0-9a-fA-F]+)""" + bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-7])""" + + escape_sequence = \ + r"""(\\("""+simple_escape+'|'+decimal_escape+'|'+hex_escape+'))' + + # string literals (K&R2: A.2.6) + string_char = r"""([^"\\\n]|""" + escape_sequence + ')' + string_literal = '"' + string_char + '*"' + bad_string_literal = '"' + string_char + '*' + bad_escape + string_char + '*"' + + # floating constants (K&R2: A.2.5.3) + exponent_part = r"""([eE][-+]?[0-9]+)""" + fractional_constant = r"""([0-9]*\.[0-9]+)|([0-9]+\.)""" + floating_constant = \ + '(((('+fractional_constant+')'+ \ + exponent_part+'?)|([0-9]+'+exponent_part+')))' + + # Ordinals + ordinal = r'@[0-9]+' + missing_ordinal_value = r'@' + # Don't allow ordinal values in octal (even invalid octal, like 09) or + # hexadecimal. + octal_or_hex_ordinal_disallowed = ( + r'@((0[0-9]+)|(' + hex_prefix + hex_digits + '))') + + ## + ## Rules for the normal state + ## + t_ignore = ' \t\r' + + # Newlines + def t_NEWLINE(self, t): + r'\n+' + t.lexer.lineno += len(t.value) + + # Operators + t_MINUS = r'-' + t_PLUS = r'\+' + t_QSTN = r'\?' + + # = + t_EQUALS = r'=' + + # => + t_RESPONSE = r'=>' + + # Delimiters + t_LPAREN = r'\(' + t_RPAREN = r'\)' + t_LBRACKET = r'\[' + t_RBRACKET = r'\]' + t_LBRACE = r'\{' + t_RBRACE = r'\}' + t_LANGLE = r'<' + t_RANGLE = r'>' + t_COMMA = r',' + t_DOT = r'\.' + t_SEMI = r';' + + t_STRING_LITERAL = string_literal + + # The following floating and integer constants are defined as + # functions to impose a strict order (otherwise, decimal + # is placed before the others because its regex is longer, + # and this is bad) + # + @TOKEN(floating_constant) + def t_FLOAT_CONST(self, t): + return t + + @TOKEN(hex_constant) + def t_INT_CONST_HEX(self, t): + return t + + @TOKEN(octal_constant_disallowed) + def t_OCTAL_CONSTANT_DISALLOWED(self, t): + msg = "Octal values not allowed" + self._error(msg, t) + + @TOKEN(decimal_constant) + def t_INT_CONST_DEC(self, t): + return t + + # unmatched string literals are caught by the preprocessor + + @TOKEN(bad_string_literal) + def t_BAD_STRING_LITERAL(self, t): + msg = "String contains invalid escape code" + self._error(msg, t) + + # Handle ordinal-related tokens in the right order: + @TOKEN(octal_or_hex_ordinal_disallowed) + def t_OCTAL_OR_HEX_ORDINAL_DISALLOWED(self, t): + msg = "Octal and hexadecimal ordinal values not allowed" + self._error(msg, t) + + @TOKEN(ordinal) + def t_ORDINAL(self, t): + return t + + @TOKEN(missing_ordinal_value) + def t_BAD_ORDINAL(self, t): + msg = "Missing ordinal value" + self._error(msg, t) + + @TOKEN(identifier) + def t_NAME(self, t): + t.type = self.keyword_map.get(t.value, "NAME") + return t + + # Ignore C and C++ style comments + def t_COMMENT(self, t): + r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)' + t.lexer.lineno += t.value.count("\n") + + def t_error(self, t): + msg = "Illegal character %s" % repr(t.value[0]) + self._error(msg, t) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/lexer_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/lexer_unittest.py new file mode 100644 index 00000000..bc9f8354 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/lexer_unittest.py @@ -0,0 +1,194 @@ +# Copyright 2014 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import importlib.util +import os.path +import sys +import unittest + +def _GetDirAbove(dirname): + """Returns the directory "above" this file containing |dirname| (which must + also be "above" this file).""" + path = os.path.abspath(__file__) + while True: + path, tail = os.path.split(path) + assert tail + if tail == dirname: + return path + +sys.path.insert(1, os.path.join(_GetDirAbove("mojo"), "third_party")) +from ply import lex + +try: + importlib.util.find_spec("mojom") +except ImportError: + sys.path.append(os.path.join(_GetDirAbove("pylib"), "pylib")) +import mojom.parse.lexer + +# This (monkey-patching LexToken to make comparison value-based) is evil, but +# we'll do it anyway. (I'm pretty sure ply's lexer never cares about comparing +# for object identity.) +def _LexTokenEq(self, other): + return self.type == other.type and self.value == other.value and \ + self.lineno == other.lineno and self.lexpos == other.lexpos + + +setattr(lex.LexToken, '__eq__', _LexTokenEq) + + +def _MakeLexToken(token_type, value, lineno=1, lexpos=0): + """Makes a LexToken with the given parameters. (Note that lineno is 1-based, + but lexpos is 0-based.)""" + rv = lex.LexToken() + rv.type, rv.value, rv.lineno, rv.lexpos = token_type, value, lineno, lexpos + return rv + + +def _MakeLexTokenForKeyword(keyword, **kwargs): + """Makes a LexToken for the given keyword.""" + return _MakeLexToken(keyword.upper(), keyword.lower(), **kwargs) + + +class LexerTest(unittest.TestCase): + """Tests |mojom.parse.lexer.Lexer|.""" + + def __init__(self, *args, **kwargs): + unittest.TestCase.__init__(self, *args, **kwargs) + # Clone all lexer instances from this one, since making a lexer is slow. + self._zygote_lexer = lex.lex(mojom.parse.lexer.Lexer("my_file.mojom")) + + def testValidKeywords(self): + """Tests valid keywords.""" + self.assertEquals( + self._SingleTokenForInput("handle"), _MakeLexTokenForKeyword("handle")) + self.assertEquals( + self._SingleTokenForInput("import"), _MakeLexTokenForKeyword("import")) + self.assertEquals( + self._SingleTokenForInput("module"), _MakeLexTokenForKeyword("module")) + self.assertEquals( + self._SingleTokenForInput("struct"), _MakeLexTokenForKeyword("struct")) + self.assertEquals( + self._SingleTokenForInput("union"), _MakeLexTokenForKeyword("union")) + self.assertEquals( + self._SingleTokenForInput("interface"), + _MakeLexTokenForKeyword("interface")) + self.assertEquals( + self._SingleTokenForInput("enum"), _MakeLexTokenForKeyword("enum")) + self.assertEquals( + self._SingleTokenForInput("const"), _MakeLexTokenForKeyword("const")) + self.assertEquals( + self._SingleTokenForInput("true"), _MakeLexTokenForKeyword("true")) + self.assertEquals( + self._SingleTokenForInput("false"), _MakeLexTokenForKeyword("false")) + self.assertEquals( + self._SingleTokenForInput("default"), + _MakeLexTokenForKeyword("default")) + self.assertEquals( + self._SingleTokenForInput("array"), _MakeLexTokenForKeyword("array")) + self.assertEquals( + self._SingleTokenForInput("map"), _MakeLexTokenForKeyword("map")) + self.assertEquals( + self._SingleTokenForInput("associated"), + _MakeLexTokenForKeyword("associated")) + + def testValidIdentifiers(self): + """Tests identifiers.""" + self.assertEquals( + self._SingleTokenForInput("abcd"), _MakeLexToken("NAME", "abcd")) + self.assertEquals( + self._SingleTokenForInput("AbC_d012_"), + _MakeLexToken("NAME", "AbC_d012_")) + self.assertEquals( + self._SingleTokenForInput("_0123"), _MakeLexToken("NAME", "_0123")) + + def testInvalidIdentifiers(self): + with self.assertRaisesRegexp( + mojom.parse.lexer.LexError, + r"^my_file\.mojom:1: Error: Illegal character '\$'$"): + self._TokensForInput("$abc") + with self.assertRaisesRegexp( + mojom.parse.lexer.LexError, + r"^my_file\.mojom:1: Error: Illegal character '\$'$"): + self._TokensForInput("a$bc") + + def testDecimalIntegerConstants(self): + self.assertEquals( + self._SingleTokenForInput("0"), _MakeLexToken("INT_CONST_DEC", "0")) + self.assertEquals( + self._SingleTokenForInput("1"), _MakeLexToken("INT_CONST_DEC", "1")) + self.assertEquals( + self._SingleTokenForInput("123"), _MakeLexToken("INT_CONST_DEC", "123")) + self.assertEquals( + self._SingleTokenForInput("10"), _MakeLexToken("INT_CONST_DEC", "10")) + + def testValidTokens(self): + """Tests valid tokens (which aren't tested elsewhere).""" + # Keywords tested in |testValidKeywords|. + # NAME tested in |testValidIdentifiers|. + self.assertEquals( + self._SingleTokenForInput("@123"), _MakeLexToken("ORDINAL", "@123")) + self.assertEquals( + self._SingleTokenForInput("456"), _MakeLexToken("INT_CONST_DEC", "456")) + self.assertEquals( + self._SingleTokenForInput("0x01aB2eF3"), + _MakeLexToken("INT_CONST_HEX", "0x01aB2eF3")) + self.assertEquals( + self._SingleTokenForInput("123.456"), + _MakeLexToken("FLOAT_CONST", "123.456")) + self.assertEquals( + self._SingleTokenForInput("\"hello\""), + _MakeLexToken("STRING_LITERAL", "\"hello\"")) + self.assertEquals( + self._SingleTokenForInput("+"), _MakeLexToken("PLUS", "+")) + self.assertEquals( + self._SingleTokenForInput("-"), _MakeLexToken("MINUS", "-")) + self.assertEquals( + self._SingleTokenForInput("?"), _MakeLexToken("QSTN", "?")) + self.assertEquals( + self._SingleTokenForInput("="), _MakeLexToken("EQUALS", "=")) + self.assertEquals( + self._SingleTokenForInput("=>"), _MakeLexToken("RESPONSE", "=>")) + self.assertEquals( + self._SingleTokenForInput("("), _MakeLexToken("LPAREN", "(")) + self.assertEquals( + self._SingleTokenForInput(")"), _MakeLexToken("RPAREN", ")")) + self.assertEquals( + self._SingleTokenForInput("["), _MakeLexToken("LBRACKET", "[")) + self.assertEquals( + self._SingleTokenForInput("]"), _MakeLexToken("RBRACKET", "]")) + self.assertEquals( + self._SingleTokenForInput("{"), _MakeLexToken("LBRACE", "{")) + self.assertEquals( + self._SingleTokenForInput("}"), _MakeLexToken("RBRACE", "}")) + self.assertEquals( + self._SingleTokenForInput("<"), _MakeLexToken("LANGLE", "<")) + self.assertEquals( + self._SingleTokenForInput(">"), _MakeLexToken("RANGLE", ">")) + self.assertEquals( + self._SingleTokenForInput(";"), _MakeLexToken("SEMI", ";")) + self.assertEquals( + self._SingleTokenForInput(","), _MakeLexToken("COMMA", ",")) + self.assertEquals(self._SingleTokenForInput("."), _MakeLexToken("DOT", ".")) + + def _TokensForInput(self, input_string): + """Gets a list of tokens for the given input string.""" + lexer = self._zygote_lexer.clone() + lexer.input(input_string) + rv = [] + while True: + tok = lexer.token() + if not tok: + return rv + rv.append(tok) + + def _SingleTokenForInput(self, input_string): + """Gets the single token for the given input string. (Raises an exception if + the input string does not result in exactly one token.)""" + toks = self._TokensForInput(input_string) + assert len(toks) == 1 + return toks[0] + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/parser.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/parser.py new file mode 100644 index 00000000..1dffd98b --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/parser.py @@ -0,0 +1,510 @@ +# Copyright 2014 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Generates a syntax tree from a Mojo IDL file.""" + +# Breaking parser stanzas is unhelpful so allow longer lines. +# pylint: disable=line-too-long + +import os.path +import sys + +from mojom import fileutil +from mojom.error import Error +from mojom.parse import ast +from mojom.parse.lexer import Lexer + +fileutil.AddLocalRepoThirdPartyDirToModulePath() +from ply import lex +from ply import yacc + +_MAX_ORDINAL_VALUE = 0xffffffff +_MAX_ARRAY_SIZE = 0xffffffff + + +class ParseError(Error): + """Class for errors from the parser.""" + + def __init__(self, filename, message, lineno=None, snippet=None): + Error.__init__( + self, + filename, + message, + lineno=lineno, + addenda=([snippet] if snippet else None)) + + +# We have methods which look like they could be functions: +# pylint: disable=R0201 +class Parser: + def __init__(self, lexer, source, filename): + self.tokens = lexer.tokens + self.source = source + self.filename = filename + + # Names of functions + # + # In general, we name functions after the left-hand-side of the rule(s) that + # they handle. E.g., |p_foo_bar| for a rule |foo_bar : ...|. + # + # There may be multiple functions handling rules for the same left-hand-side; + # then we name the functions |p_foo_bar_N| (for left-hand-side |foo_bar|), + # where N is a number (numbered starting from 1). Note that using multiple + # functions is actually more efficient than having single functions handle + # multiple rules (and, e.g., distinguishing them by examining |len(p)|). + # + # It's also possible to have a function handling multiple rules with different + # left-hand-sides. We do not do this. + # + # See http://www.dabeaz.com/ply/ply.html#ply_nn25 for more details. + + # TODO(vtl): Get rid of the braces in the module "statement". (Consider + # renaming "module" -> "package".) Then we'll be able to have a single rule + # for root (by making module "optional"). + def p_root_1(self, p): + """root : """ + p[0] = ast.Mojom(None, ast.ImportList(), []) + + def p_root_2(self, p): + """root : root module""" + if p[1].module is not None: + raise ParseError( + self.filename, + "Multiple \"module\" statements not allowed:", + p[2].lineno, + snippet=self._GetSnippet(p[2].lineno)) + if p[1].import_list.items or p[1].definition_list: + raise ParseError( + self.filename, + "\"module\" statements must precede imports and definitions:", + p[2].lineno, + snippet=self._GetSnippet(p[2].lineno)) + p[0] = p[1] + p[0].module = p[2] + + def p_root_3(self, p): + """root : root import""" + if p[1].definition_list: + raise ParseError( + self.filename, + "\"import\" statements must precede definitions:", + p[2].lineno, + snippet=self._GetSnippet(p[2].lineno)) + p[0] = p[1] + p[0].import_list.Append(p[2]) + + def p_root_4(self, p): + """root : root definition""" + p[0] = p[1] + p[0].definition_list.append(p[2]) + + def p_import(self, p): + """import : attribute_section IMPORT STRING_LITERAL SEMI""" + # 'eval' the literal to strip the quotes. + # TODO(vtl): This eval is dubious. We should unquote/unescape ourselves. + p[0] = ast.Import( + p[1], eval(p[3]), filename=self.filename, lineno=p.lineno(2)) + + def p_module(self, p): + """module : attribute_section MODULE identifier_wrapped SEMI""" + p[0] = ast.Module(p[3], p[1], filename=self.filename, lineno=p.lineno(2)) + + def p_definition(self, p): + """definition : struct + | union + | interface + | enum + | const + | feature""" + p[0] = p[1] + + def p_attribute_section_1(self, p): + """attribute_section : """ + p[0] = None + + def p_attribute_section_2(self, p): + """attribute_section : LBRACKET attribute_list RBRACKET""" + p[0] = p[2] + + def p_attribute_list_1(self, p): + """attribute_list : """ + p[0] = ast.AttributeList() + + def p_attribute_list_2(self, p): + """attribute_list : nonempty_attribute_list""" + p[0] = p[1] + + def p_nonempty_attribute_list_1(self, p): + """nonempty_attribute_list : attribute""" + p[0] = ast.AttributeList(p[1]) + + def p_nonempty_attribute_list_2(self, p): + """nonempty_attribute_list : nonempty_attribute_list COMMA attribute""" + p[0] = p[1] + p[0].Append(p[3]) + + def p_attribute_1(self, p): + """attribute : name_wrapped EQUALS identifier_wrapped""" + p[0] = ast.Attribute(p[1], + p[3][1], + filename=self.filename, + lineno=p.lineno(1)) + + def p_attribute_2(self, p): + """attribute : name_wrapped EQUALS evaled_literal + | name_wrapped EQUALS name_wrapped""" + p[0] = ast.Attribute(p[1], p[3], filename=self.filename, lineno=p.lineno(1)) + + def p_attribute_3(self, p): + """attribute : name_wrapped""" + p[0] = ast.Attribute(p[1], True, filename=self.filename, lineno=p.lineno(1)) + + def p_evaled_literal(self, p): + """evaled_literal : literal""" + # 'eval' the literal to strip the quotes. Handle keywords "true" and "false" + # specially since they cannot directly be evaluated to python boolean + # values. + if p[1] == "true": + p[0] = True + elif p[1] == "false": + p[0] = False + else: + p[0] = eval(p[1]) + + def p_struct_1(self, p): + """struct : attribute_section STRUCT name_wrapped LBRACE struct_body RBRACE SEMI""" + p[0] = ast.Struct(p[3], p[1], p[5]) + + def p_struct_2(self, p): + """struct : attribute_section STRUCT name_wrapped SEMI""" + p[0] = ast.Struct(p[3], p[1], None) + + def p_struct_body_1(self, p): + """struct_body : """ + p[0] = ast.StructBody() + + def p_struct_body_2(self, p): + """struct_body : struct_body const + | struct_body enum + | struct_body struct_field""" + p[0] = p[1] + p[0].Append(p[2]) + + def p_struct_field(self, p): + """struct_field : attribute_section typename name_wrapped ordinal default SEMI""" + p[0] = ast.StructField(p[3], p[1], p[4], p[2], p[5]) + + def p_feature(self, p): + """feature : attribute_section FEATURE NAME LBRACE feature_body RBRACE SEMI""" + p[0] = ast.Feature(p[3], p[1], p[5]) + + def p_feature_body_1(self, p): + """feature_body : """ + p[0] = ast.FeatureBody() + + def p_feature_body_2(self, p): + """feature_body : feature_body const""" + p[0] = p[1] + p[0].Append(p[2]) + + def p_union(self, p): + """union : attribute_section UNION name_wrapped LBRACE union_body RBRACE SEMI""" + p[0] = ast.Union(p[3], p[1], p[5]) + + def p_union_body_1(self, p): + """union_body : """ + p[0] = ast.UnionBody() + + def p_union_body_2(self, p): + """union_body : union_body union_field""" + p[0] = p[1] + p[1].Append(p[2]) + + def p_union_field(self, p): + """union_field : attribute_section typename name_wrapped ordinal SEMI""" + p[0] = ast.UnionField(p[3], p[1], p[4], p[2]) + + def p_default_1(self, p): + """default : """ + p[0] = None + + def p_default_2(self, p): + """default : EQUALS constant""" + p[0] = p[2] + + def p_interface(self, p): + """interface : attribute_section INTERFACE name_wrapped LBRACE interface_body RBRACE SEMI""" + p[0] = ast.Interface(p[3], p[1], p[5]) + + def p_interface_body_1(self, p): + """interface_body : """ + p[0] = ast.InterfaceBody() + + def p_interface_body_2(self, p): + """interface_body : interface_body const + | interface_body enum + | interface_body method""" + p[0] = p[1] + p[0].Append(p[2]) + + def p_response_1(self, p): + """response : """ + p[0] = None + + def p_response_2(self, p): + """response : RESPONSE LPAREN parameter_list RPAREN""" + p[0] = p[3] + + def p_method(self, p): + """method : attribute_section name_wrapped ordinal LPAREN parameter_list RPAREN response SEMI""" + p[0] = ast.Method(p[2], p[1], p[3], p[5], p[7]) + + def p_parameter_list_1(self, p): + """parameter_list : """ + p[0] = ast.ParameterList() + + def p_parameter_list_2(self, p): + """parameter_list : nonempty_parameter_list""" + p[0] = p[1] + + def p_nonempty_parameter_list_1(self, p): + """nonempty_parameter_list : parameter""" + p[0] = ast.ParameterList(p[1]) + + def p_nonempty_parameter_list_2(self, p): + """nonempty_parameter_list : nonempty_parameter_list COMMA parameter""" + p[0] = p[1] + p[0].Append(p[3]) + + def p_parameter(self, p): + """parameter : attribute_section typename name_wrapped ordinal""" + p[0] = ast.Parameter( + p[3], p[1], p[4], p[2], filename=self.filename, lineno=p.lineno(3)) + + def p_typename(self, p): + """typename : nonnullable_typename QSTN + | nonnullable_typename""" + if len(p) == 2: + p[0] = p[1] + else: + p[0] = p[1] + "?" + + def p_nonnullable_typename(self, p): + """nonnullable_typename : basictypename + | array + | fixed_array + | associative_array""" + p[0] = p[1] + + def p_basictypename(self, p): + """basictypename : remotetype + | receivertype + | associatedremotetype + | associatedreceivertype + | identifier + | ASSOCIATED identifier + | handletype""" + if len(p) == 2: + p[0] = p[1] + else: + p[0] = "asso<" + p[2] + ">" + + def p_remotetype(self, p): + """remotetype : PENDING_REMOTE LANGLE identifier RANGLE""" + p[0] = "rmt<%s>" % p[3] + + def p_receivertype(self, p): + """receivertype : PENDING_RECEIVER LANGLE identifier RANGLE""" + p[0] = "rcv<%s>" % p[3] + + def p_associatedremotetype(self, p): + """associatedremotetype : PENDING_ASSOCIATED_REMOTE LANGLE identifier RANGLE""" + p[0] = "rma<%s>" % p[3] + + def p_associatedreceivertype(self, p): + """associatedreceivertype : PENDING_ASSOCIATED_RECEIVER LANGLE identifier RANGLE""" + p[0] = "rca<%s>" % p[3] + + def p_handletype(self, p): + """handletype : HANDLE + | HANDLE LANGLE name_wrapped RANGLE""" + if len(p) == 2: + p[0] = p[1] + else: + if p[3] not in ('data_pipe_consumer', 'data_pipe_producer', + 'message_pipe', 'shared_buffer', 'platform'): + # Note: We don't enable tracking of line numbers for everything, so we + # can't use |p.lineno(3)|. + raise ParseError( + self.filename, + "Invalid handle type %r:" % p[3], + lineno=p.lineno(1), + snippet=self._GetSnippet(p.lineno(1))) + p[0] = "handle<" + p[3] + ">" + + def p_array(self, p): + """array : ARRAY LANGLE typename RANGLE""" + p[0] = p[3] + "[]" + + def p_fixed_array(self, p): + """fixed_array : ARRAY LANGLE typename COMMA INT_CONST_DEC RANGLE""" + value = int(p[5]) + if value == 0 or value > _MAX_ARRAY_SIZE: + raise ParseError( + self.filename, + "Fixed array size %d invalid:" % value, + lineno=p.lineno(5), + snippet=self._GetSnippet(p.lineno(5))) + p[0] = p[3] + "[" + p[5] + "]" + + def p_associative_array(self, p): + """associative_array : MAP LANGLE identifier COMMA typename RANGLE""" + p[0] = p[5] + "{" + p[3] + "}" + + def p_ordinal_1(self, p): + """ordinal : """ + p[0] = None + + def p_ordinal_2(self, p): + """ordinal : ORDINAL""" + value = int(p[1][1:]) + if value > _MAX_ORDINAL_VALUE: + raise ParseError( + self.filename, + "Ordinal value %d too large:" % value, + lineno=p.lineno(1), + snippet=self._GetSnippet(p.lineno(1))) + p[0] = ast.Ordinal(value, filename=self.filename, lineno=p.lineno(1)) + + def p_enum_1(self, p): + """enum : attribute_section ENUM name_wrapped LBRACE enum_value_list RBRACE SEMI + | attribute_section ENUM name_wrapped LBRACE \ + nonempty_enum_value_list COMMA RBRACE SEMI""" + p[0] = ast.Enum( + p[3], p[1], p[5], filename=self.filename, lineno=p.lineno(2)) + + def p_enum_2(self, p): + """enum : attribute_section ENUM name_wrapped SEMI""" + p[0] = ast.Enum( + p[3], p[1], None, filename=self.filename, lineno=p.lineno(2)) + + def p_enum_value_list_1(self, p): + """enum_value_list : """ + p[0] = ast.EnumValueList() + + def p_enum_value_list_2(self, p): + """enum_value_list : nonempty_enum_value_list""" + p[0] = p[1] + + def p_nonempty_enum_value_list_1(self, p): + """nonempty_enum_value_list : enum_value""" + p[0] = ast.EnumValueList(p[1]) + + def p_nonempty_enum_value_list_2(self, p): + """nonempty_enum_value_list : nonempty_enum_value_list COMMA enum_value""" + p[0] = p[1] + p[0].Append(p[3]) + + def p_enum_value(self, p): + """enum_value : attribute_section name_wrapped + | attribute_section name_wrapped EQUALS int + | attribute_section name_wrapped EQUALS identifier_wrapped""" + p[0] = ast.EnumValue( + p[2], + p[1], + p[4] if len(p) == 5 else None, + filename=self.filename, + lineno=p.lineno(2)) + + def p_const(self, p): + """const : attribute_section CONST typename name_wrapped EQUALS constant SEMI""" + p[0] = ast.Const(p[4], p[1], p[3], p[6]) + + def p_constant(self, p): + """constant : literal + | identifier_wrapped""" + p[0] = p[1] + + def p_identifier_wrapped(self, p): + """identifier_wrapped : identifier""" + p[0] = ('IDENTIFIER', p[1]) + + # TODO(vtl): Make this produce a "wrapped" identifier (probably as an + # |ast.Identifier|, to be added) and get rid of identifier_wrapped. + def p_identifier(self, p): + """identifier : name_wrapped + | name_wrapped DOT identifier""" + p[0] = ''.join(p[1:]) + + # Allow 'feature' to be a name literal not just a keyword. + def p_name_wrapped(self, p): + """name_wrapped : NAME + | FEATURE""" + p[0] = p[1] + + def p_literal(self, p): + """literal : int + | float + | TRUE + | FALSE + | DEFAULT + | STRING_LITERAL""" + p[0] = p[1] + + def p_int(self, p): + """int : int_const + | PLUS int_const + | MINUS int_const""" + p[0] = ''.join(p[1:]) + + def p_int_const(self, p): + """int_const : INT_CONST_DEC + | INT_CONST_HEX""" + p[0] = p[1] + + def p_float(self, p): + """float : FLOAT_CONST + | PLUS FLOAT_CONST + | MINUS FLOAT_CONST""" + p[0] = ''.join(p[1:]) + + def p_error(self, e): + if e is None: + # Unexpected EOF. + # TODO(vtl): Can we figure out what's missing? + raise ParseError(self.filename, "Unexpected end of file") + + if e.value == 'feature': + raise ParseError(self.filename, + "`feature` is reserved for a future mojom keyword", + lineno=e.lineno, + snippet=self._GetSnippet(e.lineno)) + + raise ParseError( + self.filename, + "Unexpected %r:" % e.value, + lineno=e.lineno, + snippet=self._GetSnippet(e.lineno)) + + def _GetSnippet(self, lineno): + return self.source.split('\n')[lineno - 1] + + +def Parse(source, filename): + """Parse source file to AST. + + Args: + source: The source text as a str (Python 2 or 3) or unicode (Python 2). + filename: The filename that |source| originates from. + + Returns: + The AST as a mojom.parse.ast.Mojom object. + """ + lexer = Lexer(filename) + parser = Parser(lexer, source, filename) + + lex.lex(object=lexer) + yacc.yacc(module=parser, debug=0, write_tables=0) + + tree = yacc.parse(source) + return tree diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/parser_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/parser_unittest.py new file mode 100644 index 00000000..0a26307b --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom/parse/parser_unittest.py @@ -0,0 +1,1375 @@ +# Copyright 2014 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import unittest + +from mojom.parse import ast +from mojom.parse import lexer +from mojom.parse import parser + +class ParserTest(unittest.TestCase): + """Tests |parser.Parse()|.""" + + def testTrivialValidSource(self): + """Tests a trivial, but valid, .mojom source.""" + + source = """\ + // This is a comment. + + module my_module; + """ + expected = ast.Mojom( + ast.Module(('IDENTIFIER', 'my_module'), None), ast.ImportList(), []) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testSourceWithCrLfs(self): + """Tests a .mojom source with CR-LFs instead of LFs.""" + + source = "// This is a comment.\r\n\r\nmodule my_module;\r\n" + expected = ast.Mojom( + ast.Module(('IDENTIFIER', 'my_module'), None), ast.ImportList(), []) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testUnexpectedEOF(self): + """Tests a "truncated" .mojom source.""" + + source = """\ + // This is a comment. + + module my_module + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom: Error: Unexpected end of file$"): + parser.Parse(source, "my_file.mojom") + + def testCommentLineNumbers(self): + """Tests that line numbers are correctly tracked when comments are + present.""" + + source1 = """\ + // Isolated C++-style comments. + + // Foo. + asdf1 + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:4: Error: Unexpected 'asdf1':\n *asdf1$"): + parser.Parse(source1, "my_file.mojom") + + source2 = """\ + // Consecutive C++-style comments. + // Foo. + // Bar. + + struct Yada { // Baz. + // Quux. + int32 x; + }; + + asdf2 + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:10: Error: Unexpected 'asdf2':\n *asdf2$"): + parser.Parse(source2, "my_file.mojom") + + source3 = """\ + /* Single-line C-style comments. */ + /* Foobar. */ + + /* Baz. */ + asdf3 + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:5: Error: Unexpected 'asdf3':\n *asdf3$"): + parser.Parse(source3, "my_file.mojom") + + source4 = """\ + /* Multi-line C-style comments. + */ + /* + Foo. + Bar. + */ + + /* Baz + Quux. */ + asdf4 + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:10: Error: Unexpected 'asdf4':\n *asdf4$"): + parser.Parse(source4, "my_file.mojom") + + def testSimpleStruct(self): + """Tests a simple .mojom source that just defines a struct.""" + + source = """\ + module my_module; + + struct MyStruct { + int32 a; + double b; + }; + """ + expected = ast.Mojom( + ast.Module(('IDENTIFIER', 'my_module'), None), ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody([ + ast.StructField('a', None, None, 'int32', None), + ast.StructField('b', None, None, 'double', None) + ])) + ]) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testSimpleStructWithoutModule(self): + """Tests a simple struct without an explict module statement.""" + + source = """\ + struct MyStruct { + int32 a; + double b; + }; + """ + expected = ast.Mojom(None, ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody([ + ast.StructField('a', None, None, 'int32', None), + ast.StructField('b', None, None, 'double', None) + ])) + ]) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testValidStructDefinitions(self): + """Tests all types of definitions that can occur in a struct.""" + + source = """\ + struct MyStruct { + enum MyEnum { VALUE }; + const double kMyConst = 1.23; + int32 a; + SomeOtherStruct b; // Invalidity detected at another stage. + }; + """ + expected = ast.Mojom(None, ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody([ + ast.Enum('MyEnum', None, + ast.EnumValueList(ast.EnumValue('VALUE', None, None))), + ast.Const('kMyConst', None, 'double', '1.23'), + ast.StructField('a', None, None, 'int32', None), + ast.StructField('b', None, None, 'SomeOtherStruct', None) + ])) + ]) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testInvalidStructDefinitions(self): + """Tests that definitions that aren't allowed in a struct are correctly + detected.""" + + source1 = """\ + struct MyStruct { + MyMethod(int32 a); + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected '\(':\n" + r" *MyMethod\(int32 a\);$"): + parser.Parse(source1, "my_file.mojom") + + source2 = """\ + struct MyStruct { + struct MyInnerStruct { + int32 a; + }; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected 'struct':\n" + r" *struct MyInnerStruct {$"): + parser.Parse(source2, "my_file.mojom") + + source3 = """\ + struct MyStruct { + interface MyInterface { + MyMethod(int32 a); + }; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:2: Error: Unexpected 'interface':\n" + r" *interface MyInterface {$"): + parser.Parse(source3, "my_file.mojom") + + def testMissingModuleName(self): + """Tests an (invalid) .mojom with a missing module name.""" + + source1 = """\ + // Missing module name. + module ; + struct MyStruct { + int32 a; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:2: Error: Unexpected ';':\n *module ;$"): + parser.Parse(source1, "my_file.mojom") + + # Another similar case, but make sure that line-number tracking/reporting + # is correct. + source2 = """\ + module + // This line intentionally left unblank. + + struct MyStruct { + int32 a; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:4: Error: Unexpected 'struct':\n" + r" *struct MyStruct {$"): + parser.Parse(source2, "my_file.mojom") + + def testMultipleModuleStatements(self): + """Tests an (invalid) .mojom with multiple module statements.""" + + source = """\ + module foo; + module bar; + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:2: Error: Multiple \"module\" statements not " + r"allowed:\n *module bar;$"): + parser.Parse(source, "my_file.mojom") + + def testModuleStatementAfterImport(self): + """Tests an (invalid) .mojom with a module statement after an import.""" + + source = """\ + import "foo.mojom"; + module foo; + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:2: Error: \"module\" statements must precede imports " + r"and definitions:\n *module foo;$"): + parser.Parse(source, "my_file.mojom") + + def testModuleStatementAfterDefinition(self): + """Tests an (invalid) .mojom with a module statement after a definition.""" + + source = """\ + struct MyStruct { + int32 a; + }; + module foo; + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:4: Error: \"module\" statements must precede imports " + r"and definitions:\n *module foo;$"): + parser.Parse(source, "my_file.mojom") + + def testImportStatementAfterDefinition(self): + """Tests an (invalid) .mojom with an import statement after a definition.""" + + source = """\ + struct MyStruct { + int32 a; + }; + import "foo.mojom"; + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:4: Error: \"import\" statements must precede " + r"definitions:\n *import \"foo.mojom\";$"): + parser.Parse(source, "my_file.mojom") + + def testEnums(self): + """Tests that enum statements are correctly parsed.""" + + source = """\ + module my_module; + enum MyEnum1 { VALUE1, VALUE2 }; // No trailing comma. + enum MyEnum2 { + VALUE1 = -1, + VALUE2 = 0, + VALUE3 = + 987, // Check that space is allowed. + VALUE4 = 0xAF12, + VALUE5 = -0x09bcd, + VALUE6 = VALUE5, + VALUE7, // Leave trailing comma. + }; + """ + expected = ast.Mojom( + ast.Module(('IDENTIFIER', 'my_module'), None), ast.ImportList(), [ + ast.Enum( + 'MyEnum1', None, + ast.EnumValueList([ + ast.EnumValue('VALUE1', None, None), + ast.EnumValue('VALUE2', None, None) + ])), + ast.Enum( + 'MyEnum2', None, + ast.EnumValueList([ + ast.EnumValue('VALUE1', None, '-1'), + ast.EnumValue('VALUE2', None, '0'), + ast.EnumValue('VALUE3', None, '+987'), + ast.EnumValue('VALUE4', None, '0xAF12'), + ast.EnumValue('VALUE5', None, '-0x09bcd'), + ast.EnumValue('VALUE6', None, ('IDENTIFIER', 'VALUE5')), + ast.EnumValue('VALUE7', None, None) + ])) + ]) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testInvalidEnumInitializers(self): + """Tests that invalid enum initializers are correctly detected.""" + + # Floating point value. + source2 = "enum MyEnum { VALUE = 0.123 };" + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:1: Error: Unexpected '0\.123':\n" + r"enum MyEnum { VALUE = 0\.123 };$"): + parser.Parse(source2, "my_file.mojom") + + # Boolean value. + source2 = "enum MyEnum { VALUE = true };" + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:1: Error: Unexpected 'true':\n" + r"enum MyEnum { VALUE = true };$"): + parser.Parse(source2, "my_file.mojom") + + def testConsts(self): + """Tests some constants and struct members initialized with them.""" + + source = """\ + module my_module; + + struct MyStruct { + const int8 kNumber = -1; + int8 number@0 = kNumber; + }; + """ + expected = ast.Mojom( + ast.Module(('IDENTIFIER', 'my_module'), None), ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody([ + ast.Const('kNumber', None, 'int8', '-1'), + ast.StructField('number', None, ast.Ordinal(0), 'int8', + ('IDENTIFIER', 'kNumber')) + ])) + ]) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testNoConditionals(self): + """Tests that ?: is not allowed.""" + + source = """\ + module my_module; + + enum MyEnum { + MY_ENUM_1 = 1 ? 2 : 3 + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:4: Error: Unexpected '\?':\n" + r" *MY_ENUM_1 = 1 \? 2 : 3$"): + parser.Parse(source, "my_file.mojom") + + def testSimpleOrdinals(self): + """Tests that (valid) ordinal values are scanned correctly.""" + + source = """\ + module my_module; + + // This isn't actually valid .mojom, but the problem (missing ordinals) + // should be handled at a different level. + struct MyStruct { + int32 a0@0; + int32 a1@1; + int32 a2@2; + int32 a9@9; + int32 a10 @10; + int32 a11 @11; + int32 a29 @29; + int32 a1234567890 @1234567890; + }; + """ + expected = ast.Mojom( + ast.Module(('IDENTIFIER', 'my_module'), None), ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody([ + ast.StructField('a0', None, ast.Ordinal(0), 'int32', None), + ast.StructField('a1', None, ast.Ordinal(1), 'int32', None), + ast.StructField('a2', None, ast.Ordinal(2), 'int32', None), + ast.StructField('a9', None, ast.Ordinal(9), 'int32', None), + ast.StructField('a10', None, ast.Ordinal(10), 'int32', + None), + ast.StructField('a11', None, ast.Ordinal(11), 'int32', + None), + ast.StructField('a29', None, ast.Ordinal(29), 'int32', + None), + ast.StructField('a1234567890', None, + ast.Ordinal(1234567890), 'int32', None) + ])) + ]) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testInvalidOrdinals(self): + """Tests that (lexically) invalid ordinals are correctly detected.""" + + source1 = """\ + module my_module; + + struct MyStruct { + int32 a_missing@; + }; + """ + with self.assertRaisesRegexp( + lexer.LexError, r"^my_file\.mojom:4: Error: Missing ordinal value$"): + parser.Parse(source1, "my_file.mojom") + + source2 = """\ + module my_module; + + struct MyStruct { + int32 a_octal@01; + }; + """ + with self.assertRaisesRegexp( + lexer.LexError, r"^my_file\.mojom:4: Error: " + r"Octal and hexadecimal ordinal values not allowed$"): + parser.Parse(source2, "my_file.mojom") + + source3 = """\ + module my_module; struct MyStruct { int32 a_invalid_octal@08; }; + """ + with self.assertRaisesRegexp( + lexer.LexError, r"^my_file\.mojom:1: Error: " + r"Octal and hexadecimal ordinal values not allowed$"): + parser.Parse(source3, "my_file.mojom") + + source4 = "module my_module; struct MyStruct { int32 a_hex@0x1aB9; };" + with self.assertRaisesRegexp( + lexer.LexError, r"^my_file\.mojom:1: Error: " + r"Octal and hexadecimal ordinal values not allowed$"): + parser.Parse(source4, "my_file.mojom") + + source5 = "module my_module; struct MyStruct { int32 a_hex@0X0; };" + with self.assertRaisesRegexp( + lexer.LexError, r"^my_file\.mojom:1: Error: " + r"Octal and hexadecimal ordinal values not allowed$"): + parser.Parse(source5, "my_file.mojom") + + source6 = """\ + struct MyStruct { + int32 a_too_big@999999999999; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: " + r"Ordinal value 999999999999 too large:\n" + r" *int32 a_too_big@999999999999;$"): + parser.Parse(source6, "my_file.mojom") + + def testNestedNamespace(self): + """Tests that "nested" namespaces work.""" + + source = """\ + module my.mod; + + struct MyStruct { + int32 a; + }; + """ + expected = ast.Mojom( + ast.Module(('IDENTIFIER', 'my.mod'), None), ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody(ast.StructField('a', None, None, 'int32', None))) + ]) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testValidHandleTypes(self): + """Tests (valid) handle types.""" + + source = """\ + struct MyStruct { + handle a; + handle b; + handle c; + handle < message_pipe > d; + handle + < shared_buffer + > e; + handle + f; + }; + """ + expected = ast.Mojom(None, ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody([ + ast.StructField('a', None, None, 'handle', None), + ast.StructField('b', None, None, 'handle', + None), + ast.StructField('c', None, None, 'handle', + None), + ast.StructField('d', None, None, 'handle', None), + ast.StructField('e', None, None, 'handle', None), + ast.StructField('f', None, None, 'handle', None) + ])) + ]) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testInvalidHandleType(self): + """Tests an invalid (unknown) handle type.""" + + source = """\ + struct MyStruct { + handle foo; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: " + r"Invalid handle type 'wtf_is_this':\n" + r" *handle foo;$"): + parser.Parse(source, "my_file.mojom") + + def testValidDefaultValues(self): + """Tests default values that are valid (to the parser).""" + + source = """\ + struct MyStruct { + int16 a0 = 0; + uint16 a1 = 0x0; + uint16 a2 = 0x00; + uint16 a3 = 0x01; + uint16 a4 = 0xcd; + int32 a5 = 12345; + int64 a6 = -12345; + int64 a7 = +12345; + uint32 a8 = 0x12cd3; + uint32 a9 = -0x12cD3; + uint32 a10 = +0x12CD3; + bool a11 = true; + bool a12 = false; + float a13 = 1.2345; + float a14 = -1.2345; + float a15 = +1.2345; + float a16 = 123.; + float a17 = .123; + double a18 = 1.23E10; + double a19 = 1.E-10; + double a20 = .5E+10; + double a21 = -1.23E10; + double a22 = +.123E10; + }; + """ + expected = ast.Mojom(None, ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody([ + ast.StructField('a0', None, None, 'int16', '0'), + ast.StructField('a1', None, None, 'uint16', '0x0'), + ast.StructField('a2', None, None, 'uint16', '0x00'), + ast.StructField('a3', None, None, 'uint16', '0x01'), + ast.StructField('a4', None, None, 'uint16', '0xcd'), + ast.StructField('a5', None, None, 'int32', '12345'), + ast.StructField('a6', None, None, 'int64', '-12345'), + ast.StructField('a7', None, None, 'int64', '+12345'), + ast.StructField('a8', None, None, 'uint32', '0x12cd3'), + ast.StructField('a9', None, None, 'uint32', '-0x12cD3'), + ast.StructField('a10', None, None, 'uint32', '+0x12CD3'), + ast.StructField('a11', None, None, 'bool', 'true'), + ast.StructField('a12', None, None, 'bool', 'false'), + ast.StructField('a13', None, None, 'float', '1.2345'), + ast.StructField('a14', None, None, 'float', '-1.2345'), + ast.StructField('a15', None, None, 'float', '+1.2345'), + ast.StructField('a16', None, None, 'float', '123.'), + ast.StructField('a17', None, None, 'float', '.123'), + ast.StructField('a18', None, None, 'double', '1.23E10'), + ast.StructField('a19', None, None, 'double', '1.E-10'), + ast.StructField('a20', None, None, 'double', '.5E+10'), + ast.StructField('a21', None, None, 'double', '-1.23E10'), + ast.StructField('a22', None, None, 'double', '+.123E10') + ])) + ]) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testValidFixedSizeArray(self): + """Tests parsing a fixed size array.""" + + source = """\ + struct MyStruct { + array normal_array; + array fixed_size_array_one_entry; + array fixed_size_array_ten_entries; + array>, 2> nested_arrays; + }; + """ + expected = ast.Mojom(None, ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody([ + ast.StructField('normal_array', None, None, 'int32[]', None), + ast.StructField('fixed_size_array_one_entry', None, None, + 'int32[1]', None), + ast.StructField('fixed_size_array_ten_entries', None, None, + 'int32[10]', None), + ast.StructField('nested_arrays', None, None, 'int32[1][][2]', + None) + ])) + ]) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testValidNestedArray(self): + """Tests parsing a nested array.""" + + source = "struct MyStruct { array> nested_array; };" + expected = ast.Mojom(None, ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody( + ast.StructField('nested_array', None, None, 'int32[][]', None))) + ]) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testInvalidFixedArraySize(self): + """Tests that invalid fixed array bounds are correctly detected.""" + + source1 = """\ + struct MyStruct { + array zero_size_array; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:2: Error: Fixed array size 0 invalid:\n" + r" *array zero_size_array;$"): + parser.Parse(source1, "my_file.mojom") + + source2 = """\ + struct MyStruct { + array too_big_array; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:2: Error: Fixed array size 999999999999 invalid:\n" + r" *array too_big_array;$"): + parser.Parse(source2, "my_file.mojom") + + source3 = """\ + struct MyStruct { + array not_a_number; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected 'abcdefg':\n" + r" *array not_a_number;"): + parser.Parse(source3, "my_file.mojom") + + def testValidAssociativeArrays(self): + """Tests that we can parse valid associative array structures.""" + + source1 = "struct MyStruct { map data; };" + expected1 = ast.Mojom(None, ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody( + [ast.StructField('data', None, None, 'uint8{string}', None)])) + ]) + self.assertEquals(parser.Parse(source1, "my_file.mojom"), expected1) + + source2 = "interface MyInterface { MyMethod(map a); };" + expected2 = ast.Mojom(None, ast.ImportList(), [ + ast.Interface( + 'MyInterface', None, + ast.InterfaceBody( + ast.Method( + 'MyMethod', None, None, + ast.ParameterList( + ast.Parameter('a', None, None, 'uint8{string}')), + None))) + ]) + self.assertEquals(parser.Parse(source2, "my_file.mojom"), expected2) + + source3 = "struct MyStruct { map> data; };" + expected3 = ast.Mojom(None, ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody( + [ast.StructField('data', None, None, 'uint8[]{string}', None)])) + ]) + self.assertEquals(parser.Parse(source3, "my_file.mojom"), expected3) + + def testValidMethod(self): + """Tests parsing method declarations.""" + + source1 = "interface MyInterface { MyMethod(int32 a); };" + expected1 = ast.Mojom(None, ast.ImportList(), [ + ast.Interface( + 'MyInterface', None, + ast.InterfaceBody( + ast.Method( + 'MyMethod', None, None, + ast.ParameterList(ast.Parameter('a', None, None, 'int32')), + None))) + ]) + self.assertEquals(parser.Parse(source1, "my_file.mojom"), expected1) + + source2 = """\ + interface MyInterface { + MyMethod1@0(int32 a@0, int64 b@1); + MyMethod2@1() => (); + }; + """ + expected2 = ast.Mojom(None, ast.ImportList(), [ + ast.Interface( + 'MyInterface', None, + ast.InterfaceBody([ + ast.Method( + 'MyMethod1', None, ast.Ordinal(0), + ast.ParameterList([ + ast.Parameter('a', None, ast.Ordinal(0), 'int32'), + ast.Parameter('b', None, ast.Ordinal(1), 'int64') + ]), None), + ast.Method('MyMethod2', None, ast.Ordinal(1), + ast.ParameterList(), ast.ParameterList()) + ])) + ]) + self.assertEquals(parser.Parse(source2, "my_file.mojom"), expected2) + + source3 = """\ + interface MyInterface { + MyMethod(string a) => (int32 a, bool b); + }; + """ + expected3 = ast.Mojom(None, ast.ImportList(), [ + ast.Interface( + 'MyInterface', None, + ast.InterfaceBody( + ast.Method( + 'MyMethod', None, None, + ast.ParameterList(ast.Parameter('a', None, None, 'string')), + ast.ParameterList([ + ast.Parameter('a', None, None, 'int32'), + ast.Parameter('b', None, None, 'bool') + ])))) + ]) + self.assertEquals(parser.Parse(source3, "my_file.mojom"), expected3) + + def testInvalidMethods(self): + """Tests that invalid method declarations are correctly detected.""" + + # No trailing commas. + source1 = """\ + interface MyInterface { + MyMethod(string a,); + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected '\)':\n" + r" *MyMethod\(string a,\);$"): + parser.Parse(source1, "my_file.mojom") + + # No leading commas. + source2 = """\ + interface MyInterface { + MyMethod(, string a); + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected ',':\n" + r" *MyMethod\(, string a\);$"): + parser.Parse(source2, "my_file.mojom") + + def testValidInterfaceDefinitions(self): + """Tests all types of definitions that can occur in an interface.""" + + source = """\ + interface MyInterface { + enum MyEnum { VALUE }; + const int32 kMyConst = 123; + MyMethod(int32 x) => (MyEnum y); + }; + """ + expected = ast.Mojom(None, ast.ImportList(), [ + ast.Interface( + 'MyInterface', None, + ast.InterfaceBody([ + ast.Enum('MyEnum', None, + ast.EnumValueList(ast.EnumValue('VALUE', None, None))), + ast.Const('kMyConst', None, 'int32', '123'), + ast.Method( + 'MyMethod', None, None, + ast.ParameterList(ast.Parameter('x', None, None, 'int32')), + ast.ParameterList(ast.Parameter('y', None, None, 'MyEnum'))) + ])) + ]) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testInvalidInterfaceDefinitions(self): + """Tests that definitions that aren't allowed in an interface are correctly + detected.""" + + source1 = """\ + interface MyInterface { + struct MyStruct { + int32 a; + }; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected 'struct':\n" + r" *struct MyStruct {$"): + parser.Parse(source1, "my_file.mojom") + + source2 = """\ + interface MyInterface { + interface MyInnerInterface { + MyMethod(int32 x); + }; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:2: Error: Unexpected 'interface':\n" + r" *interface MyInnerInterface {$"): + parser.Parse(source2, "my_file.mojom") + + source3 = """\ + interface MyInterface { + int32 my_field; + }; + """ + # The parser thinks that "int32" is a plausible name for a method, so it's + # "my_field" that gives it away. + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected 'my_field':\n" + r" *int32 my_field;$"): + parser.Parse(source3, "my_file.mojom") + + def testValidAttributes(self): + """Tests parsing attributes (and attribute lists).""" + + # Note: We use structs because they have (optional) attribute lists. + + # Empty attribute list. + source1 = "[] struct MyStruct {};" + expected1 = ast.Mojom( + None, ast.ImportList(), + [ast.Struct('MyStruct', ast.AttributeList(), ast.StructBody())]) + self.assertEquals(parser.Parse(source1, "my_file.mojom"), expected1) + + # One-element attribute list, with name value. + source2 = "[MyAttribute=MyName] struct MyStruct {};" + expected2 = ast.Mojom(None, ast.ImportList(), [ + ast.Struct('MyStruct', + ast.AttributeList(ast.Attribute("MyAttribute", "MyName")), + ast.StructBody()) + ]) + self.assertEquals(parser.Parse(source2, "my_file.mojom"), expected2) + + # Two-element attribute list, with one string value and one integer value. + source3 = "[MyAttribute1 = \"hello\", MyAttribute2 = 5] struct MyStruct {};" + expected3 = ast.Mojom(None, ast.ImportList(), [ + ast.Struct( + 'MyStruct', + ast.AttributeList([ + ast.Attribute("MyAttribute1", "hello"), + ast.Attribute("MyAttribute2", 5) + ]), ast.StructBody()) + ]) + self.assertEquals(parser.Parse(source3, "my_file.mojom"), expected3) + + # Various places that attribute list is allowed. + source4 = """\ + [Attr0=0] module my_module; + + [Attr1=1] import "my_import"; + + [Attr2=2] struct MyStruct { + [Attr3=3] int32 a; + }; + [Attr4=4] union MyUnion { + [Attr5=5] int32 a; + }; + [Attr6=6] enum MyEnum { + [Attr7=7] a + }; + [Attr8=8] interface MyInterface { + [Attr9=9] MyMethod([Attr10=10] int32 a) => ([Attr11=11] bool b); + }; + [Attr12=12] const double kMyConst = 1.23; + """ + expected4 = ast.Mojom( + ast.Module(('IDENTIFIER', 'my_module'), + ast.AttributeList([ast.Attribute("Attr0", 0)])), + ast.ImportList( + ast.Import( + ast.AttributeList([ast.Attribute("Attr1", 1)]), "my_import")), + [ + ast.Struct( + 'MyStruct', ast.AttributeList(ast.Attribute("Attr2", 2)), + ast.StructBody( + ast.StructField( + 'a', ast.AttributeList([ast.Attribute("Attr3", 3)]), + None, 'int32', None))), + ast.Union( + 'MyUnion', ast.AttributeList(ast.Attribute("Attr4", 4)), + ast.UnionBody( + ast.UnionField( + 'a', ast.AttributeList([ast.Attribute("Attr5", 5)]), + None, 'int32'))), + ast.Enum( + 'MyEnum', ast.AttributeList(ast.Attribute("Attr6", 6)), + ast.EnumValueList( + ast.EnumValue( + 'VALUE', ast.AttributeList([ast.Attribute("Attr7", 7)]), + None))), + ast.Interface( + 'MyInterface', ast.AttributeList(ast.Attribute("Attr8", 8)), + ast.InterfaceBody( + ast.Method( + 'MyMethod', ast.AttributeList( + ast.Attribute("Attr9", 9)), None, + ast.ParameterList( + ast.Parameter( + 'a', + ast.AttributeList([ast.Attribute("Attr10", 10) + ]), None, 'int32')), + ast.ParameterList( + ast.Parameter( + 'b', + ast.AttributeList([ast.Attribute("Attr11", 11) + ]), None, 'bool'))))), + ast.Const('kMyConst', ast.AttributeList( + ast.Attribute("Attr12", 12)), 'double', '1.23') + ]) + self.assertEquals(parser.Parse(source4, "my_file.mojom"), expected4) + + # TODO(vtl): Boolean attributes don't work yet. (In fact, we just |eval()| + # literal (non-name) values, which is extremely dubious.) + + def testInvalidAttributes(self): + """Tests that invalid attributes and attribute lists are correctly + detected.""" + + # Trailing commas not allowed. + source1 = "[MyAttribute=MyName,] struct MyStruct {};" + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:1: Error: Unexpected '\]':\n" + r"\[MyAttribute=MyName,\] struct MyStruct {};$"): + parser.Parse(source1, "my_file.mojom") + + # Missing value. + source2 = "[MyAttribute=] struct MyStruct {};" + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:1: Error: Unexpected '\]':\n" + r"\[MyAttribute=\] struct MyStruct {};$"): + parser.Parse(source2, "my_file.mojom") + + # Missing key. + source3 = "[=MyName] struct MyStruct {};" + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:1: Error: Unexpected '=':\n" + r"\[=MyName\] struct MyStruct {};$"): + parser.Parse(source3, "my_file.mojom") + + def testValidImports(self): + """Tests parsing import statements.""" + + # One import (no module statement). + source1 = "import \"somedir/my.mojom\";" + expected1 = ast.Mojom(None, + ast.ImportList(ast.Import(None, "somedir/my.mojom")), + []) + self.assertEquals(parser.Parse(source1, "my_file.mojom"), expected1) + + # Two imports (no module statement). + source2 = """\ + import "somedir/my1.mojom"; + import "somedir/my2.mojom"; + """ + expected2 = ast.Mojom( + None, + ast.ImportList([ + ast.Import(None, "somedir/my1.mojom"), + ast.Import(None, "somedir/my2.mojom") + ]), []) + self.assertEquals(parser.Parse(source2, "my_file.mojom"), expected2) + + # Imports with module statement. + source3 = """\ + module my_module; + import "somedir/my1.mojom"; + import "somedir/my2.mojom"; + """ + expected3 = ast.Mojom( + ast.Module(('IDENTIFIER', 'my_module'), None), + ast.ImportList([ + ast.Import(None, "somedir/my1.mojom"), + ast.Import(None, "somedir/my2.mojom") + ]), []) + self.assertEquals(parser.Parse(source3, "my_file.mojom"), expected3) + + def testInvalidImports(self): + """Tests that invalid import statements are correctly detected.""" + + source1 = """\ + // Make the error occur on line 2. + import invalid + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected 'invalid':\n" + r" *import invalid$"): + parser.Parse(source1, "my_file.mojom") + + source2 = """\ + import // Missing string. + struct MyStruct { + int32 a; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected 'struct':\n" + r" *struct MyStruct {$"): + parser.Parse(source2, "my_file.mojom") + + source3 = """\ + import "foo.mojom" // Missing semicolon. + struct MyStruct { + int32 a; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected 'struct':\n" + r" *struct MyStruct {$"): + parser.Parse(source3, "my_file.mojom") + + def testValidNullableTypes(self): + """Tests parsing nullable types.""" + + source = """\ + struct MyStruct { + int32? a; // This is actually invalid, but handled at a different + // level. + string? b; + array ? c; + array ? d; + array?>? e; + array? f; + array? g; + some_struct? h; + handle? i; + handle? j; + handle? k; + handle? l; + handle? m; + pending_receiver? n; + handle? o; + }; + """ + expected = ast.Mojom(None, ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody([ + ast.StructField('a', None, None, 'int32?', None), + ast.StructField('b', None, None, 'string?', None), + ast.StructField('c', None, None, 'int32[]?', None), + ast.StructField('d', None, None, 'string?[]?', None), + ast.StructField('e', None, None, 'int32[]?[]?', None), + ast.StructField('f', None, None, 'int32[1]?', None), + ast.StructField('g', None, None, 'string?[1]?', None), + ast.StructField('h', None, None, 'some_struct?', None), + ast.StructField('i', None, None, 'handle?', None), + ast.StructField('j', None, None, 'handle?', + None), + ast.StructField('k', None, None, 'handle?', + None), + ast.StructField('l', None, None, 'handle?', None), + ast.StructField('m', None, None, 'handle?', + None), + ast.StructField('n', None, None, 'rcv?', None), + ast.StructField('o', None, None, 'handle?', None) + ])) + ]) + self.assertEquals(parser.Parse(source, "my_file.mojom"), expected) + + def testInvalidNullableTypes(self): + """Tests that invalid nullable types are correctly detected.""" + source1 = """\ + struct MyStruct { + string?? a; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected '\?':\n" + r" *string\?\? a;$"): + parser.Parse(source1, "my_file.mojom") + + source2 = """\ + struct MyStruct { + handle? a; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected '<':\n" + r" *handle\? a;$"): + parser.Parse(source2, "my_file.mojom") + + def testSimpleUnion(self): + """Tests a simple .mojom source that just defines a union.""" + source = """\ + module my_module; + + union MyUnion { + int32 a; + double b; + }; + """ + expected = ast.Mojom( + ast.Module(('IDENTIFIER', 'my_module'), None), ast.ImportList(), [ + ast.Union( + 'MyUnion', None, + ast.UnionBody([ + ast.UnionField('a', None, None, 'int32'), + ast.UnionField('b', None, None, 'double') + ])) + ]) + actual = parser.Parse(source, "my_file.mojom") + self.assertEquals(actual, expected) + + def testUnionWithOrdinals(self): + """Test that ordinals are assigned to fields.""" + source = """\ + module my_module; + + union MyUnion { + int32 a @10; + double b @30; + }; + """ + expected = ast.Mojom( + ast.Module(('IDENTIFIER', 'my_module'), None), ast.ImportList(), [ + ast.Union( + 'MyUnion', None, + ast.UnionBody([ + ast.UnionField('a', None, ast.Ordinal(10), 'int32'), + ast.UnionField('b', None, ast.Ordinal(30), 'double') + ])) + ]) + actual = parser.Parse(source, "my_file.mojom") + self.assertEquals(actual, expected) + + def testUnionWithStructMembers(self): + """Test that struct members are accepted.""" + source = """\ + module my_module; + + union MyUnion { + SomeStruct s; + }; + """ + expected = ast.Mojom( + ast.Module(('IDENTIFIER', 'my_module'), None), ast.ImportList(), [ + ast.Union( + 'MyUnion', None, + ast.UnionBody([ast.UnionField('s', None, None, 'SomeStruct')])) + ]) + actual = parser.Parse(source, "my_file.mojom") + self.assertEquals(actual, expected) + + def testUnionWithArrayMember(self): + """Test that array members are accepted.""" + source = """\ + module my_module; + + union MyUnion { + array a; + }; + """ + expected = ast.Mojom( + ast.Module(('IDENTIFIER', 'my_module'), None), ast.ImportList(), [ + ast.Union( + 'MyUnion', None, + ast.UnionBody([ast.UnionField('a', None, None, 'int32[]')])) + ]) + actual = parser.Parse(source, "my_file.mojom") + self.assertEquals(actual, expected) + + def testUnionWithMapMember(self): + """Test that map members are accepted.""" + source = """\ + module my_module; + + union MyUnion { + map m; + }; + """ + expected = ast.Mojom( + ast.Module(('IDENTIFIER', 'my_module'), None), ast.ImportList(), [ + ast.Union( + 'MyUnion', None, + ast.UnionBody( + [ast.UnionField('m', None, None, 'string{int32}')])) + ]) + actual = parser.Parse(source, "my_file.mojom") + self.assertEquals(actual, expected) + + def testUnionDisallowNestedStruct(self): + """Tests that structs cannot be nested in unions.""" + source = """\ + module my_module; + + union MyUnion { + struct MyStruct { + int32 a; + }; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:4: Error: Unexpected 'struct':\n" + r" *struct MyStruct {$"): + parser.Parse(source, "my_file.mojom") + + def testUnionDisallowNestedInterfaces(self): + """Tests that interfaces cannot be nested in unions.""" + source = """\ + module my_module; + + union MyUnion { + interface MyInterface { + MyMethod(int32 a); + }; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:4: Error: Unexpected 'interface':\n" + r" *interface MyInterface {$"): + parser.Parse(source, "my_file.mojom") + + def testUnionDisallowNestedUnion(self): + """Tests that unions cannot be nested in unions.""" + source = """\ + module my_module; + + union MyUnion { + union MyOtherUnion { + int32 a; + }; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:4: Error: Unexpected 'union':\n" + r" *union MyOtherUnion {$"): + parser.Parse(source, "my_file.mojom") + + def testUnionDisallowNestedEnum(self): + """Tests that enums cannot be nested in unions.""" + source = """\ + module my_module; + + union MyUnion { + enum MyEnum { + A, + }; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:4: Error: Unexpected 'enum':\n" + r" *enum MyEnum {$"): + parser.Parse(source, "my_file.mojom") + + def testValidAssociatedKinds(self): + """Tests parsing associated interfaces and requests.""" + source1 = """\ + struct MyStruct { + associated MyInterface a; + pending_associated_receiver b; + associated MyInterface? c; + pending_associated_receiver? d; + }; + """ + expected1 = ast.Mojom(None, ast.ImportList(), [ + ast.Struct( + 'MyStruct', None, + ast.StructBody([ + ast.StructField('a', None, None, 'asso', None), + ast.StructField('b', None, None, 'rca', None), + ast.StructField('c', None, None, 'asso?', None), + ast.StructField('d', None, None, 'rca?', None) + ])) + ]) + self.assertEquals(parser.Parse(source1, "my_file.mojom"), expected1) + + source2 = """\ + interface MyInterface { + MyMethod(associated A a) =>(pending_associated_receiver b); + };""" + expected2 = ast.Mojom(None, ast.ImportList(), [ + ast.Interface( + 'MyInterface', None, + ast.InterfaceBody( + ast.Method( + 'MyMethod', None, None, + ast.ParameterList(ast.Parameter('a', None, None, + 'asso')), + ast.ParameterList(ast.Parameter('b', None, None, + 'rca'))))) + ]) + self.assertEquals(parser.Parse(source2, "my_file.mojom"), expected2) + + def testInvalidAssociatedKinds(self): + """Tests that invalid associated interfaces and requests are correctly + detected.""" + source1 = """\ + struct MyStruct { + associated associated SomeInterface a; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, + r"^my_file\.mojom:2: Error: Unexpected 'associated':\n" + r" *associated associated SomeInterface a;$"): + parser.Parse(source1, "my_file.mojom") + + source2 = """\ + struct MyStruct { + associated handle a; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected 'handle':\n" + r" *associated handle a;$"): + parser.Parse(source2, "my_file.mojom") + + source3 = """\ + struct MyStruct { + associated? MyInterface& a; + }; + """ + with self.assertRaisesRegexp( + parser.ParseError, r"^my_file\.mojom:2: Error: Unexpected '\?':\n" + r" *associated\? MyInterface& a;$"): + parser.Parse(source3, "my_file.mojom") + +if __name__ == "__main__": + unittest.main() diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser.py new file mode 100755 index 00000000..9693090e --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser.py @@ -0,0 +1,502 @@ +#!/usr/bin/env python3 +# Copyright 2020 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Parses mojom IDL files. + +This script parses one or more input mojom files and produces corresponding +module files fully describing the definitions contained within each mojom. The +module data is pickled and can be easily consumed by other tools to, e.g., +generate usable language bindings. +""" + +import argparse +import builtins +import codecs +import errno +import json +import logging +import multiprocessing +import os +import os.path +import sys +import traceback +from collections import defaultdict + +from mojom.generate import module +from mojom.generate import translate +from mojom.parse import parser +from mojom.parse import conditional_features + + +# Disable this for easier debugging. +_ENABLE_MULTIPROCESSING = True + +# https://docs.python.org/3/library/multiprocessing.html#:~:text=bpo-33725 +if __name__ == '__main__' and sys.platform == 'darwin': + multiprocessing.set_start_method('fork') +_MULTIPROCESSING_USES_FORK = multiprocessing.get_start_method() == 'fork' + + +def _ResolveRelativeImportPath(path, roots): + """Attempts to resolve a relative import path against a set of possible roots. + + Args: + path: The relative import path to resolve. + roots: A list of absolute paths which will be checked in descending length + order for a match against path. + + Returns: + A normalized absolute path combining one of the roots with the input path if + and only if such a file exists. + + Raises: + ValueError: The path could not be resolved against any of the given roots. + """ + for root in reversed(sorted(roots, key=len)): + abs_path = os.path.join(root, path) + if os.path.isfile(abs_path): + return os.path.normcase(os.path.normpath(abs_path)) + + raise ValueError('"%s" does not exist in any of %s' % (path, roots)) + + +def RebaseAbsolutePath(path, roots): + """Rewrites an absolute file path as relative to an absolute directory path in + roots. + + Args: + path: The absolute path of an existing file. + roots: A list of absolute directory paths. The given path argument must fall + within one of these directories. + + Returns: + A path equivalent to the input path, but relative to one of the provided + roots. If the input path falls within multiple roots, the longest root is + chosen (and thus the shortest relative path is returned). + + Paths returned by this method always use forward slashes as a separator to + mirror mojom import syntax. + + Raises: + ValueError if the given path does not fall within any of the listed roots. + """ + assert os.path.isabs(path) + assert os.path.isfile(path) + assert all(map(os.path.isabs, roots)) + + sorted_roots = list(reversed(sorted(roots, key=len))) + + def try_rebase_path(path, root): + head, rebased_path = os.path.split(path) + while head != root: + head, tail = os.path.split(head) + if not tail: + return None + rebased_path = os.path.join(tail, rebased_path) + return rebased_path + + for root in sorted_roots: + relative_path = try_rebase_path(path, root) + if relative_path: + # TODO(crbug.com/953884): Use pathlib for this kind of thing once we're + # fully migrated to Python 3. + return relative_path.replace('\\', '/') + + raise ValueError('%s does not fall within any of %s' % (path, sorted_roots)) + + +def _GetModuleFilename(mojom_filename): + return mojom_filename + '-module' + + +def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts, + dependencies, loaded_modules, module_metadata): + """Recursively ensures that a module and its dependencies are loaded. + + Args: + mojom_abspath: An absolute file path pointing to a mojom file to load. + module_path: The relative path used to identify mojom_abspath. + abs_paths: A mapping from module paths to absolute file paths for all + inputs given to this execution of the script. + asts: A map from each input mojom's absolute path to its parsed AST. + dependencies: A mapping of which input mojoms depend on each other, indexed + by absolute file path. + loaded_modules: A mapping of all modules loaded so far, including non-input + modules that were pulled in as transitive dependencies of the inputs. + module_metadata: Metadata to be attached to every module loaded by this + helper. + + Returns: + None + + On return, loaded_modules will be populated with the loaded input mojom's + Module as well as the Modules of all of its transitive dependencies.""" + + if mojom_abspath in loaded_modules: + # Already done. + return + + for dep_abspath, dep_path in sorted(dependencies[mojom_abspath]): + if dep_abspath not in loaded_modules: + _EnsureInputLoaded(dep_abspath, dep_path, abs_paths, asts, dependencies, + loaded_modules, module_metadata) + + imports = {} + for imp in asts[mojom_abspath].import_list: + path = imp.import_filename + imports[path] = loaded_modules[abs_paths[path]] + loaded_modules[mojom_abspath] = translate.OrderedModule( + asts[mojom_abspath], module_path, imports) + loaded_modules[mojom_abspath].metadata = dict(module_metadata) + + +def _CollectAllowedImportsFromBuildMetadata(build_metadata_filename): + allowed_imports = set() + processed_deps = set() + + def collect(metadata_filename): + processed_deps.add(metadata_filename) + + # Paths in the metadata file are relative to the metadata file's dir. + metadata_dir = os.path.abspath(os.path.dirname(metadata_filename)) + + def to_abs(s): + return os.path.normpath(os.path.join(metadata_dir, s)) + + with open(metadata_filename) as f: + metadata = json.load(f) + allowed_imports.update( + [os.path.normcase(to_abs(s)) for s in metadata['sources']]) + for dep_metadata in metadata['deps']: + dep_metadata = to_abs(dep_metadata) + if dep_metadata not in processed_deps: + collect(dep_metadata) + + collect(build_metadata_filename) + return allowed_imports + + +# multiprocessing helper. +def _ParseAstHelper(mojom_abspath, enabled_features): + with codecs.open(mojom_abspath, encoding='utf-8') as f: + ast = parser.Parse(f.read(), mojom_abspath) + conditional_features.RemoveDisabledDefinitions(ast, enabled_features) + return mojom_abspath, ast + + +# multiprocessing helper. +def _SerializeHelper(mojom_abspath, mojom_path): + module_path = os.path.join(_SerializeHelper.output_root_path, + _GetModuleFilename(mojom_path)) + module_dir = os.path.dirname(module_path) + if not os.path.exists(module_dir): + try: + # Python 2 doesn't support exist_ok on makedirs(), so we just ignore + # that failure if it happens. It's possible during build due to races + # among build steps with module outputs in the same directory. + os.makedirs(module_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + with open(module_path, 'wb') as f: + _SerializeHelper.loaded_modules[mojom_abspath].Dump(f) + + +class _ExceptionWrapper: + def __init__(self): + # Do not capture exception object to ensure pickling works. + self.formatted_trace = traceback.format_exc() + + +class _FuncWrapper: + """Marshals exceptions and spreads args.""" + + def __init__(self, func): + self._func = func + + def __call__(self, args): + # multiprocessing does not gracefully handle excptions. + # https://crbug.com/1219044 + try: + return self._func(*args) + except: # pylint: disable=bare-except + return _ExceptionWrapper() + + +def _Shard(target_func, arg_list, processes=None): + arg_list = list(arg_list) + if processes is None: + processes = multiprocessing.cpu_count() + # Seems optimal to have each process perform at least 2 tasks. + processes = min(processes, len(arg_list) // 2) + + if sys.platform == 'win32': + # TODO(crbug.com/1190269) - we can't use more than 56 + # cores on Windows or Python3 may hang. + processes = min(processes, 56) + + # Don't spin up processes unless there is enough work to merit doing so. + if not _ENABLE_MULTIPROCESSING or processes < 2: + for arg_tuple in arg_list: + yield target_func(*arg_tuple) + return + + pool = multiprocessing.Pool(processes=processes) + try: + wrapped_func = _FuncWrapper(target_func) + for result in pool.imap_unordered(wrapped_func, arg_list): + if isinstance(result, _ExceptionWrapper): + sys.stderr.write(result.formatted_trace) + sys.exit(1) + yield result + finally: + pool.close() + pool.join() # Needed on Windows to avoid WindowsError during terminate. + pool.terminate() + + +def _ParseMojoms(mojom_files, + input_root_paths, + output_root_path, + module_root_paths, + enabled_features, + module_metadata, + allowed_imports=None): + """Parses a set of mojom files and produces serialized module outputs. + + Args: + mojom_files: A list of mojom files to process. Paths must be absolute paths + which fall within one of the input or output root paths. + input_root_paths: A list of absolute filesystem paths which may be used to + resolve relative mojom file paths. + output_root_path: An absolute filesystem path which will service as the root + for all emitted artifacts. Artifacts produced from a given mojom file + are based on the mojom's relative path, rebased onto this path. + Additionally, the script expects this root to contain already-generated + modules for any transitive dependencies not listed in mojom_files. + module_root_paths: A list of absolute filesystem paths which contain + already-generated modules for any non-transitive dependencies. + enabled_features: A list of enabled feature names, controlling which AST + nodes are filtered by [EnableIf] or [EnableIfNot] attributes. + module_metadata: A list of 2-tuples representing metadata key-value pairs to + attach to each compiled module output. + + Returns: + None. + + Upon completion, a mojom-module file will be saved for each input mojom. + """ + assert input_root_paths + assert output_root_path + + loaded_mojom_asts = {} + loaded_modules = {} + input_dependencies = defaultdict(set) + mojom_files_to_parse = dict((os.path.normcase(abs_path), + RebaseAbsolutePath(abs_path, input_root_paths)) + for abs_path in mojom_files) + abs_paths = dict( + (path, abs_path) for abs_path, path in mojom_files_to_parse.items()) + + logging.info('Parsing %d .mojom into ASTs', len(mojom_files_to_parse)) + map_args = ((mojom_abspath, enabled_features) + for mojom_abspath in mojom_files_to_parse) + for mojom_abspath, ast in _Shard(_ParseAstHelper, map_args): + loaded_mojom_asts[mojom_abspath] = ast + + logging.info('Processing dependencies') + for mojom_abspath, ast in sorted(loaded_mojom_asts.items()): + invalid_imports = [] + for imp in ast.import_list: + import_abspath = _ResolveRelativeImportPath(imp.import_filename, + input_root_paths) + if allowed_imports and import_abspath not in allowed_imports: + invalid_imports.append(imp.import_filename) + + abs_paths[imp.import_filename] = import_abspath + if import_abspath in mojom_files_to_parse: + # This import is in the input list, so we're going to translate it + # into a module below; however it's also a dependency of another input + # module. We retain record of dependencies to help with input + # processing later. + input_dependencies[mojom_abspath].add( + (import_abspath, imp.import_filename)) + elif import_abspath not in loaded_modules: + # We have an import that isn't being parsed right now. It must already + # be parsed and have a module file sitting in a corresponding output + # location. + module_path = _GetModuleFilename(imp.import_filename) + module_abspath = _ResolveRelativeImportPath( + module_path, module_root_paths + [output_root_path]) + with open(module_abspath, 'rb') as module_file: + loaded_modules[import_abspath] = module.Module.Load(module_file) + + if invalid_imports: + raise ValueError( + '\nThe file %s imports the following files not allowed by build ' + 'dependencies:\n\n%s\n' % (mojom_abspath, '\n'.join(invalid_imports))) + logging.info('Loaded %d modules from dependencies', len(loaded_modules)) + + # At this point all transitive imports not listed as inputs have been loaded + # and we have a complete dependency tree of the unprocessed inputs. Now we can + # load all the inputs, resolving dependencies among them recursively as we go. + logging.info('Ensuring inputs are loaded') + num_existing_modules_loaded = len(loaded_modules) + for mojom_abspath, mojom_path in mojom_files_to_parse.items(): + _EnsureInputLoaded(mojom_abspath, mojom_path, abs_paths, loaded_mojom_asts, + input_dependencies, loaded_modules, module_metadata) + assert (num_existing_modules_loaded + + len(mojom_files_to_parse) == len(loaded_modules)) + + # Now we have fully translated modules for every input and every transitive + # dependency. We can dump the modules to disk for other tools to use. + logging.info('Serializing %d modules', len(mojom_files_to_parse)) + + # Windows does not use fork() for multiprocessing, so we'd need to pass + # loaded_module via IPC rather than via globals. Doing so is slower than not + # using multiprocessing. + _SerializeHelper.loaded_modules = loaded_modules + _SerializeHelper.output_root_path = output_root_path + # Doesn't seem to help past 4. Perhaps IO bound here? + processes = 4 if _MULTIPROCESSING_USES_FORK else 0 + map_args = mojom_files_to_parse.items() + for _ in _Shard(_SerializeHelper, map_args, processes=processes): + pass + + +def Run(command_line): + debug_logging = os.environ.get('MOJOM_PARSER_DEBUG', '0') != '0' + logging.basicConfig(level=logging.DEBUG if debug_logging else logging.WARNING, + format='%(levelname).1s %(relativeCreated)6d %(message)s') + logging.info('Started (%s)', os.path.basename(sys.argv[0])) + + arg_parser = argparse.ArgumentParser( + description=""" +Parses one or more mojom files and produces corresponding module outputs fully +describing the definitions therein. The output is exhaustive, stable, and +sufficient for another tool to consume and emit e.g. usable language +bindings based on the original mojoms.""", + epilog=""" +Note that each transitive import dependency reachable from the input mojoms must +either also be listed as an input or must have its corresponding compiled module +already present in the provided output root.""") + + arg_parser.add_argument( + '--input-root', + default=[], + action='append', + metavar='ROOT', + dest='input_root_paths', + help='Adds ROOT to the set of root paths against which relative input ' + 'paths should be resolved. Provided root paths are always searched ' + 'in order from longest absolute path to shortest.') + arg_parser.add_argument( + '--output-root', + action='store', + required=True, + dest='output_root_path', + metavar='ROOT', + help='Use ROOT as the root path in which the parser should emit compiled ' + 'modules for each processed input mojom. The path of emitted module is ' + 'based on the relative input path, rebased onto this root. Note that ' + 'ROOT is also searched for existing modules of any transitive imports ' + 'which were not included in the set of inputs.') + arg_parser.add_argument( + '--module-root', + default=[], + action='append', + metavar='ROOT', + dest='module_root_paths', + help='Adds ROOT to the set of root paths to search for existing modules ' + 'of non-transitive imports. Provided root paths are always searched in ' + 'order from longest absolute path to shortest.') + arg_parser.add_argument( + '--mojoms', + nargs='+', + dest='mojom_files', + default=[], + metavar='MOJOM_FILE', + help='Input mojom filename(s). Each filename must be either an absolute ' + 'path which falls within one of the given input or output roots, or a ' + 'relative path the parser will attempt to resolve using each of those ' + 'roots in unspecified order.') + arg_parser.add_argument( + '--mojom-file-list', + action='store', + metavar='LIST_FILENAME', + help='Input file whose contents are a list of mojoms to process. This ' + 'may be provided in lieu of --mojoms to avoid hitting command line ' + 'length limtations') + arg_parser.add_argument( + '--enable-feature', + dest='enabled_features', + default=[], + action='append', + metavar='FEATURE', + help='Enables a named feature when parsing the given mojoms. Features ' + 'are identified by arbitrary string values. Specifying this flag with a ' + 'given FEATURE name will cause the parser to process any syntax elements ' + 'tagged with an [EnableIf=FEATURE] or [EnableIfNot] attribute. If this ' + 'flag is not provided for a given FEATURE, such tagged elements are ' + 'discarded by the parser and will not be present in the compiled output.') + arg_parser.add_argument( + '--check-imports', + dest='build_metadata_filename', + action='store', + metavar='METADATA_FILENAME', + help='Instructs the parser to check imports against a set of allowed ' + 'imports. Allowed imports are based on build metadata within ' + 'METADATA_FILENAME. This is a JSON file with a `sources` key listing ' + 'paths to the set of input mojom files being processed by this parser ' + 'run, and a `deps` key listing paths to metadata files for any ' + 'dependencies of these inputs. This feature can be used to implement ' + 'build-time dependency checking for mojom imports, where each build ' + 'metadata file corresponds to a build target in the dependency graph of ' + 'a typical build system.') + arg_parser.add_argument( + '--add-module-metadata', + dest='module_metadata', + default=[], + action='append', + metavar='KEY=VALUE', + help='Adds a metadata key-value pair to the output module. This can be ' + 'used by build toolchains to augment parsed mojom modules with product-' + 'specific metadata for later extraction and use by custom bindings ' + 'generators.') + + args, _ = arg_parser.parse_known_args(command_line) + if args.mojom_file_list: + with open(args.mojom_file_list) as f: + args.mojom_files.extend(f.read().split()) + + if not args.mojom_files: + raise ValueError( + 'Must list at least one mojom file via --mojoms or --mojom-file-list') + + mojom_files = list(map(os.path.abspath, args.mojom_files)) + input_roots = list(map(os.path.abspath, args.input_root_paths)) + output_root = os.path.abspath(args.output_root_path) + module_roots = list(map(os.path.abspath, args.module_root_paths)) + + if args.build_metadata_filename: + allowed_imports = _CollectAllowedImportsFromBuildMetadata( + args.build_metadata_filename) + else: + allowed_imports = None + + module_metadata = list( + map(lambda kvp: tuple(kvp.split('=')), args.module_metadata)) + _ParseMojoms(mojom_files, input_roots, output_root, module_roots, + args.enabled_features, module_metadata, allowed_imports) + logging.info('Finished') + + +if __name__ == '__main__': + Run(sys.argv[1:]) + # Exit without running GC, which can save multiple seconds due to the large + # number of object created. But flush is necessary as os._exit doesn't do + # that. + sys.stdout.flush() + sys.stderr.flush() + os._exit(0) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser_test_case.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser_test_case.py new file mode 100644 index 00000000..f0ee6966 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser_test_case.py @@ -0,0 +1,73 @@ +# Copyright 2020 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import json +import os +import os.path +import shutil +import tempfile +import unittest + +import mojom_parser + +from mojom.generate import module + + +class MojomParserTestCase(unittest.TestCase): + """Tests covering the behavior defined by the main mojom_parser.py script. + This includes behavior around input and output path manipulation, dependency + resolution, and module serialization and deserialization.""" + + def __init__(self, method_name): + super().__init__(method_name) + self._temp_dir = None + + def setUp(self): + self._temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self._temp_dir) + self._temp_dir = None + + def GetPath(self, path): + assert not os.path.isabs(path) + return os.path.join(self._temp_dir, path) + + def GetModulePath(self, path): + assert not os.path.isabs(path) + return os.path.join(self.GetPath('out'), path) + '-module' + + def WriteFile(self, path, contents): + full_path = self.GetPath(path) + dirname = os.path.dirname(full_path) + if not os.path.exists(dirname): + os.makedirs(dirname) + with open(full_path, 'w') as f: + f.write(contents) + + def LoadModule(self, mojom_path): + with open(self.GetModulePath(mojom_path), 'rb') as f: + return module.Module.Load(f) + + def ParseMojoms(self, mojoms, metadata=None): + """Parse all input mojoms relative the temp dir.""" + out_dir = self.GetPath('out') + args = [ + '--input-root', self._temp_dir, '--input-root', out_dir, + '--output-root', out_dir, '--mojoms' + ] + list(map(lambda mojom: os.path.join(self._temp_dir, mojom), mojoms)) + if metadata: + args.extend(['--check-imports', self.GetPath(metadata)]) + mojom_parser.Run(args) + + def ExtractTypes(self, mojom): + filename = 'test.mojom' + self.WriteFile(filename, mojom) + self.ParseMojoms([filename]) + m = self.LoadModule(filename) + definitions = {} + for kinds in (m.enums, m.structs, m.unions, m.interfaces, m.features): + for kind in kinds: + definitions[kind.mojom_name] = kind + return definitions diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser_unittest.py new file mode 100644 index 00000000..353a2b6e --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser_unittest.py @@ -0,0 +1,186 @@ +# Copyright 2020 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import json + +from mojom_parser_test_case import MojomParserTestCase + + +class MojomParserTest(MojomParserTestCase): + """Tests covering the behavior defined by the main mojom_parser.py script. + This includes behavior around input and output path manipulation, dependency + resolution, and module serialization and deserialization.""" + + def testBasicParse(self): + """Basic test to verify that we can parse a mojom file and get a module.""" + mojom = 'foo/bar.mojom' + self.WriteFile( + mojom, """\ + module test; + enum TestEnum { kFoo }; + """) + self.ParseMojoms([mojom]) + + m = self.LoadModule(mojom) + self.assertEqual('foo/bar.mojom', m.path) + self.assertEqual('test', m.mojom_namespace) + self.assertEqual(1, len(m.enums)) + + def testBasicParseWithAbsolutePaths(self): + """Verifies that we can parse a mojom file given an absolute path input.""" + mojom = 'foo/bar.mojom' + self.WriteFile( + mojom, """\ + module test; + enum TestEnum { kFoo }; + """) + self.ParseMojoms([self.GetPath(mojom)]) + + m = self.LoadModule(mojom) + self.assertEqual('foo/bar.mojom', m.path) + self.assertEqual('test', m.mojom_namespace) + self.assertEqual(1, len(m.enums)) + + def testImport(self): + """Verify imports within the same set of mojom inputs.""" + a = 'a.mojom' + b = 'b.mojom' + self.WriteFile( + a, """\ + module a; + import "b.mojom"; + struct Foo { b.Bar bar; };""") + self.WriteFile(b, """\ + module b; + struct Bar {};""") + self.ParseMojoms([a, b]) + + ma = self.LoadModule(a) + mb = self.LoadModule(b) + self.assertEqual('a.mojom', ma.path) + self.assertEqual('b.mojom', mb.path) + self.assertEqual(1, len(ma.imports)) + self.assertEqual(mb, ma.imports[0]) + + def testPreProcessedImport(self): + """Verify imports processed by a previous parser execution can be loaded + properly when parsing a dependent mojom.""" + a = 'a.mojom' + self.WriteFile(a, """\ + module a; + struct Bar {};""") + self.ParseMojoms([a]) + + b = 'b.mojom' + self.WriteFile( + b, """\ + module b; + import "a.mojom"; + struct Foo { a.Bar bar; };""") + self.ParseMojoms([b]) + + def testMissingImport(self): + """Verify that an import fails if the imported mojom does not exist.""" + a = 'a.mojom' + self.WriteFile( + a, """\ + module a; + import "non-existent.mojom"; + struct Bar {};""") + with self.assertRaisesRegexp(ValueError, "does not exist"): + self.ParseMojoms([a]) + + def testUnparsedImport(self): + """Verify that an import fails if the imported mojom is not in the set of + mojoms provided to the parser on this execution AND there is no pre-existing + parsed output module already on disk for it.""" + a = 'a.mojom' + b = 'b.mojom' + self.WriteFile(a, """\ + module a; + struct Bar {};""") + self.WriteFile( + b, """\ + module b; + import "a.mojom"; + struct Foo { a.Bar bar; };""") + + # a.mojom has not been parsed yet, so its import will fail when processing + # b.mojom here. + with self.assertRaisesRegexp(ValueError, "does not exist"): + self.ParseMojoms([b]) + + def testCheckImportsBasic(self): + """Verify that the parser can handle --check-imports with a valid set of + inputs, including support for transitive dependency resolution.""" + a = 'a.mojom' + a_metadata = 'out/a.build_metadata' + b = 'b.mojom' + b_metadata = 'out/b.build_metadata' + c = 'c.mojom' + c_metadata = 'out/c.build_metadata' + self.WriteFile(a_metadata, + json.dumps({ + "sources": [self.GetPath(a)], + "deps": [] + })) + self.WriteFile( + b_metadata, + json.dumps({ + "sources": [self.GetPath(b)], + "deps": [self.GetPath(a_metadata)] + })) + self.WriteFile( + c_metadata, + json.dumps({ + "sources": [self.GetPath(c)], + "deps": [self.GetPath(b_metadata)] + })) + self.WriteFile(a, """\ + module a; + struct Bar {};""") + self.WriteFile( + b, """\ + module b; + import "a.mojom"; + struct Foo { a.Bar bar; };""") + self.WriteFile( + c, """\ + module c; + import "a.mojom"; + import "b.mojom"; + struct Baz { b.Foo foo; };""") + self.ParseMojoms([a], metadata=a_metadata) + self.ParseMojoms([b], metadata=b_metadata) + self.ParseMojoms([c], metadata=c_metadata) + + def testCheckImportsMissing(self): + """Verify that the parser rejects valid input mojoms when imports don't + agree with build metadata given via --check-imports.""" + a = 'a.mojom' + a_metadata = 'out/a.build_metadata' + b = 'b.mojom' + b_metadata = 'out/b.build_metadata' + self.WriteFile(a_metadata, + json.dumps({ + "sources": [self.GetPath(a)], + "deps": [] + })) + self.WriteFile(b_metadata, + json.dumps({ + "sources": [self.GetPath(b)], + "deps": [] + })) + self.WriteFile(a, """\ + module a; + struct Bar {};""") + self.WriteFile( + b, """\ + module b; + import "a.mojom"; + struct Foo { a.Bar bar; };""") + + self.ParseMojoms([a], metadata=a_metadata) + with self.assertRaisesRegexp(ValueError, "not allowed by build"): + self.ParseMojoms([b], metadata=b_metadata) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/stable_attribute_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/stable_attribute_unittest.py new file mode 100644 index 00000000..d10d69c6 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/stable_attribute_unittest.py @@ -0,0 +1,127 @@ +# Copyright 2020 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from mojom_parser_test_case import MojomParserTestCase + +from mojom.generate import module + + +class StableAttributeTest(MojomParserTestCase): + """Tests covering usage of the [Stable] attribute.""" + + def testStableAttributeTagging(self): + """Verify that we recognize the [Stable] attribute on relevant definitions + and the resulting parser outputs are tagged accordingly.""" + mojom = 'test.mojom' + self.WriteFile( + mojom, """\ + [Stable] enum TestEnum { kFoo }; + enum UnstableEnum { kBar }; + [Stable] struct TestStruct { TestEnum a; }; + struct UnstableStruct { UnstableEnum a; }; + [Stable] union TestUnion { TestEnum a; TestStruct b; }; + union UnstableUnion { UnstableEnum a; UnstableStruct b; }; + [Stable] interface TestInterface { Foo@0(TestUnion x) => (); }; + interface UnstableInterface { Foo(UnstableUnion x) => (); }; + """) + self.ParseMojoms([mojom]) + + m = self.LoadModule(mojom) + self.assertEqual(2, len(m.enums)) + self.assertTrue(m.enums[0].stable) + self.assertFalse(m.enums[1].stable) + self.assertEqual(2, len(m.structs)) + self.assertTrue(m.structs[0].stable) + self.assertFalse(m.structs[1].stable) + self.assertEqual(2, len(m.unions)) + self.assertTrue(m.unions[0].stable) + self.assertFalse(m.unions[1].stable) + self.assertEqual(2, len(m.interfaces)) + self.assertTrue(m.interfaces[0].stable) + self.assertFalse(m.interfaces[1].stable) + + def testStableStruct(self): + """A [Stable] struct is valid if all its fields are also stable.""" + self.ExtractTypes('[Stable] struct S {};') + self.ExtractTypes('[Stable] struct S { int32 x; bool b; };') + self.ExtractTypes('[Stable] enum E { A }; [Stable] struct S { E e; };') + self.ExtractTypes('[Stable] struct S {}; [Stable] struct T { S s; };') + self.ExtractTypes( + '[Stable] struct S {}; [Stable] struct T { array ss; };') + self.ExtractTypes( + '[Stable] interface F {}; [Stable] struct T { pending_remote f; };') + + with self.assertRaisesRegexp(Exception, 'because it depends on E'): + self.ExtractTypes('enum E { A }; [Stable] struct S { E e; };') + with self.assertRaisesRegexp(Exception, 'because it depends on X'): + self.ExtractTypes('struct X {}; [Stable] struct S { X x; };') + with self.assertRaisesRegexp(Exception, 'because it depends on T'): + self.ExtractTypes('struct T {}; [Stable] struct S { array xs; };') + with self.assertRaisesRegexp(Exception, 'because it depends on T'): + self.ExtractTypes('struct T {}; [Stable] struct S { map xs; };') + with self.assertRaisesRegexp(Exception, 'because it depends on T'): + self.ExtractTypes('struct T {}; [Stable] struct S { map xs; };') + with self.assertRaisesRegexp(Exception, 'because it depends on F'): + self.ExtractTypes( + 'interface F {}; [Stable] struct S { pending_remote f; };') + with self.assertRaisesRegexp(Exception, 'because it depends on F'): + self.ExtractTypes( + 'interface F {}; [Stable] struct S { pending_receiver f; };') + + def testStableUnion(self): + """A [Stable] union is valid if all its fields' types are also stable.""" + self.ExtractTypes('[Stable] union U {};') + self.ExtractTypes('[Stable] union U { int32 x; bool b; };') + self.ExtractTypes('[Stable] enum E { A }; [Stable] union U { E e; };') + self.ExtractTypes('[Stable] struct S {}; [Stable] union U { S s; };') + self.ExtractTypes( + '[Stable] struct S {}; [Stable] union U { array ss; };') + self.ExtractTypes( + '[Stable] interface F {}; [Stable] union U { pending_remote f; };') + + with self.assertRaisesRegexp(Exception, 'because it depends on E'): + self.ExtractTypes('enum E { A }; [Stable] union U { E e; };') + with self.assertRaisesRegexp(Exception, 'because it depends on X'): + self.ExtractTypes('struct X {}; [Stable] union U { X x; };') + with self.assertRaisesRegexp(Exception, 'because it depends on T'): + self.ExtractTypes('struct T {}; [Stable] union U { array xs; };') + with self.assertRaisesRegexp(Exception, 'because it depends on T'): + self.ExtractTypes('struct T {}; [Stable] union U { map xs; };') + with self.assertRaisesRegexp(Exception, 'because it depends on T'): + self.ExtractTypes('struct T {}; [Stable] union U { map xs; };') + with self.assertRaisesRegexp(Exception, 'because it depends on F'): + self.ExtractTypes( + 'interface F {}; [Stable] union U { pending_remote f; };') + with self.assertRaisesRegexp(Exception, 'because it depends on F'): + self.ExtractTypes( + 'interface F {}; [Stable] union U { pending_receiver f; };') + + def testStableInterface(self): + """A [Stable] interface is valid if all its methods' parameter types are + stable, including response parameters where applicable.""" + self.ExtractTypes('[Stable] interface F {};') + self.ExtractTypes('[Stable] interface F { A@0(int32 x); };') + self.ExtractTypes('[Stable] interface F { A@0(int32 x) => (bool b); };') + self.ExtractTypes("""\ + [Stable] enum E { A, B, C }; + [Stable] struct S {}; + [Stable] interface F { A@0(E e, S s) => (bool b, array s); }; + """) + + with self.assertRaisesRegexp(Exception, 'because it depends on E'): + self.ExtractTypes( + 'enum E { A, B, C }; [Stable] interface F { A@0(E e); };') + with self.assertRaisesRegexp(Exception, 'because it depends on E'): + self.ExtractTypes( + 'enum E { A, B, C }; [Stable] interface F { A@0(int32 x) => (E e); };' + ) + with self.assertRaisesRegexp(Exception, 'because it depends on S'): + self.ExtractTypes( + 'struct S {}; [Stable] interface F { A@0(int32 x) => (S s); };') + with self.assertRaisesRegexp(Exception, 'because it depends on S'): + self.ExtractTypes( + 'struct S {}; [Stable] interface F { A@0(S s) => (bool b); };') + + with self.assertRaisesRegexp(Exception, 'explicit method ordinals'): + self.ExtractTypes('[Stable] interface F { A() => (); };') diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/union_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/union_unittest.py new file mode 100644 index 00000000..6b2525e5 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/union_unittest.py @@ -0,0 +1,44 @@ +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from mojom_parser_test_case import MojomParserTestCase + + +class UnionTest(MojomParserTestCase): + """Tests union parsing behavior.""" + + def testExtensibleMustHaveDefault(self): + """Verifies that extensible unions must have a default field.""" + mojom = 'foo.mojom' + self.WriteFile(mojom, 'module foo; [Extensible] union U { bool x; };') + with self.assertRaisesRegexp(Exception, 'must specify a \[Default\]'): + self.ParseMojoms([mojom]) + + def testExtensibleSingleDefault(self): + """Verifies that extensible unions must not have multiple default fields.""" + mojom = 'foo.mojom' + self.WriteFile( + mojom, """\ + module foo; + [Extensible] union U { + [Default] bool x; + [Default] bool y; + }; + """) + with self.assertRaisesRegexp(Exception, 'Multiple \[Default\] fields'): + self.ParseMojoms([mojom]) + + def testExtensibleDefaultTypeValid(self): + """Verifies that an extensible union's default field must be nullable or + integral type.""" + mojom = 'foo.mojom' + self.WriteFile( + mojom, """\ + module foo; + [Extensible] union U { + [Default] handle p; + }; + """) + with self.assertRaisesRegexp(Exception, 'must be nullable or integral'): + self.ParseMojoms([mojom]) diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py b/utils/codegen/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py new file mode 100644 index 00000000..45e45ec5 --- /dev/null +++ b/utils/codegen/ipc/mojo/public/tools/mojom/version_compatibility_unittest.py @@ -0,0 +1,458 @@ +# Copyright 2020 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from mojom.generate import module +from mojom_parser_test_case import MojomParserTestCase + + +class VersionCompatibilityTest(MojomParserTestCase): + """Tests covering compatibility between two versions of the same mojom type + definition. This coverage ensures that we can reliably detect unsafe changes + to definitions that are expected to tolerate version skew in production + environments.""" + + def _GetTypeCompatibilityMap(self, old_mojom, new_mojom): + """Helper to support the implementation of assertBackwardCompatible and + assertNotBackwardCompatible.""" + + old = self.ExtractTypes(old_mojom) + new = self.ExtractTypes(new_mojom) + self.assertEqual(set(old.keys()), set(new.keys()), + 'Old and new test mojoms should use the same type names.') + + checker = module.BackwardCompatibilityChecker() + compatibility_map = {} + for name in old: + try: + compatibility_map[name] = checker.IsBackwardCompatible( + new[name], old[name]) + except Exception: + compatibility_map[name] = False + return compatibility_map + + def assertBackwardCompatible(self, old_mojom, new_mojom): + compatibility_map = self._GetTypeCompatibilityMap(old_mojom, new_mojom) + for name, compatible in compatibility_map.items(): + if not compatible: + raise AssertionError( + 'Given the old mojom:\n\n %s\n\nand the new mojom:\n\n %s\n\n' + 'The new definition of %s should pass a backward-compatibiity ' + 'check, but it does not.' % (old_mojom, new_mojom, name)) + + def assertNotBackwardCompatible(self, old_mojom, new_mojom): + compatibility_map = self._GetTypeCompatibilityMap(old_mojom, new_mojom) + if all(compatibility_map.values()): + raise AssertionError( + 'Given the old mojom:\n\n %s\n\nand the new mojom:\n\n %s\n\n' + 'The new mojom should fail a backward-compatibility check, but it ' + 'does not.' % (old_mojom, new_mojom)) + + def testNewNonExtensibleEnumValue(self): + """Adding a value to a non-extensible enum breaks backward-compatibility.""" + self.assertNotBackwardCompatible('enum E { kFoo, kBar };', + 'enum E { kFoo, kBar, kBaz };') + + def testNewNonExtensibleEnumValueWithMinVersion(self): + """Adding a value to a non-extensible enum breaks backward-compatibility, + even with a new [MinVersion] specified for the value.""" + self.assertNotBackwardCompatible( + 'enum E { kFoo, kBar };', 'enum E { kFoo, kBar, [MinVersion=1] kBaz };') + + def testNewValueInExistingVersion(self): + """Adding a value to an existing version is not allowed, even if the old + enum was marked [Extensible]. Note that it is irrelevant whether or not the + new enum is marked [Extensible].""" + self.assertNotBackwardCompatible( + '[Extensible] enum E { [Default] kFoo, kBar };', + 'enum E { kFoo, kBar, kBaz };') + self.assertNotBackwardCompatible( + '[Extensible] enum E { [Default] kFoo, kBar };', + '[Extensible] enum E { [Default] kFoo, kBar, kBaz };') + self.assertNotBackwardCompatible( + '[Extensible] enum E { [Default] kFoo, [MinVersion=1] kBar };', + 'enum E { kFoo, [MinVersion=1] kBar, [MinVersion=1] kBaz };') + + def testEnumValueRemoval(self): + """Removal of an enum value is never valid even for [Extensible] enums.""" + self.assertNotBackwardCompatible('enum E { kFoo, kBar };', + 'enum E { kFoo };') + self.assertNotBackwardCompatible( + '[Extensible] enum E { [Default] kFoo, kBar };', + '[Extensible] enum E { [Default] kFoo };') + self.assertNotBackwardCompatible( + '[Extensible] enum E { [Default] kA, [MinVersion=1] kB };', + '[Extensible] enum E { [Default] kA, };') + self.assertNotBackwardCompatible( + """[Extensible] enum E { + [Default] kA, + [MinVersion=1] kB, + [MinVersion=1] kZ };""", + '[Extensible] enum E { [Default] kA, [MinVersion=1] kB };') + + def testNewExtensibleEnumValueWithMinVersion(self): + """Adding a new and properly [MinVersion]'d value to an [Extensible] enum + is a backward-compatible change. Note that it is irrelevant whether or not + the new enum is marked [Extensible].""" + self.assertBackwardCompatible('[Extensible] enum E { [Default] kA, kB };', + 'enum E { kA, kB, [MinVersion=1] kC };') + self.assertBackwardCompatible( + '[Extensible] enum E { [Default] kA, kB };', + '[Extensible] enum E { [Default] kA, kB, [MinVersion=1] kC };') + self.assertBackwardCompatible( + '[Extensible] enum E { [Default] kA, [MinVersion=1] kB };', + """[Extensible] enum E { + [Default] kA, + [MinVersion=1] kB, + [MinVersion=2] kC };""") + + def testRenameEnumValue(self): + """Renaming an enum value does not affect backward-compatibility. Only + numeric value is relevant.""" + self.assertBackwardCompatible('enum E { kA, kB };', 'enum E { kX, kY };') + + def testAddEnumValueAlias(self): + """Adding new enum fields does not affect backward-compatibility if it does + not introduce any new numeric values.""" + self.assertBackwardCompatible( + 'enum E { kA, kB };', 'enum E { kA, kB, kC = kA, kD = 1, kE = kD };') + + def testEnumIdentity(self): + """An unchanged enum is obviously backward-compatible.""" + self.assertBackwardCompatible('enum E { kA, kB, kC };', + 'enum E { kA, kB, kC };') + + def testNewStructFieldUnversioned(self): + """Adding a new field to a struct without a new (i.e. higher than any + existing version) [MinVersion] tag breaks backward-compatibility.""" + self.assertNotBackwardCompatible('struct S { string a; };', + 'struct S { string a; string b; };') + + def testStructFieldRemoval(self): + """Removing a field from a struct breaks backward-compatibility.""" + self.assertNotBackwardCompatible('struct S { string a; string b; };', + 'struct S { string a; };') + + def testStructFieldTypeChange(self): + """Changing the type of an existing field always breaks + backward-compatibility.""" + self.assertNotBackwardCompatible('struct S { string a; };', + 'struct S { array a; };') + + def testStructFieldBecomingOptional(self): + """Changing a field from non-optional to optional breaks + backward-compatibility.""" + self.assertNotBackwardCompatible('struct S { string a; };', + 'struct S { string? a; };') + + def testStructFieldBecomingNonOptional(self): + """Changing a field from optional to non-optional breaks + backward-compatibility.""" + self.assertNotBackwardCompatible('struct S { string? a; };', + 'struct S { string a; };') + + def testStructFieldOrderChange(self): + """Changing the order of fields breaks backward-compatibility.""" + self.assertNotBackwardCompatible('struct S { string a; bool b; };', + 'struct S { bool b; string a; };') + self.assertNotBackwardCompatible('struct S { string a@0; bool b@1; };', + 'struct S { string a@1; bool b@0; };') + + def testStructFieldMinVersionChange(self): + """Changing the MinVersion of a field breaks backward-compatibility.""" + self.assertNotBackwardCompatible( + 'struct S { string a; [MinVersion=1] string? b; };', + 'struct S { string a; [MinVersion=2] string? b; };') + + def testStructFieldTypeChange(self): + """If a struct field's own type definition changes, the containing struct + is backward-compatible if and only if the field type's change is + backward-compatible.""" + self.assertBackwardCompatible( + 'struct S {}; struct T { S s; };', + 'struct S { [MinVersion=1] int32 x; }; struct T { S s; };') + self.assertBackwardCompatible( + '[Extensible] enum E { [Default] kA }; struct S { E e; };', + """[Extensible] enum E { + [Default] kA, + [MinVersion=1] kB }; + struct S { E e; };""") + self.assertNotBackwardCompatible( + 'struct S {}; struct T { S s; };', + 'struct S { int32 x; }; struct T { S s; };') + self.assertNotBackwardCompatible( + '[Extensible] enum E { [Default] kA }; struct S { E e; };', + '[Extensible] enum E { [Default] kA, kB }; struct S { E e; };') + + def testNewStructFieldWithInvalidMinVersion(self): + """Adding a new field using an existing MinVersion breaks backward- + compatibility.""" + self.assertNotBackwardCompatible( + """\ + struct S { + string a; + [MinVersion=1] string? b; + }; + """, """\ + struct S { + string a; + [MinVersion=1] string? b; + [MinVersion=1] string? c; + };""") + + def testNewStructFieldWithValidMinVersion(self): + """Adding a new field is safe if tagged with a MinVersion greater than any + previously used MinVersion in the struct.""" + self.assertBackwardCompatible( + 'struct S { int32 a; };', + 'struct S { int32 a; [MinVersion=1] int32 b; };') + self.assertBackwardCompatible( + 'struct S { int32 a; [MinVersion=1] int32 b; };', + 'struct S { int32 a; [MinVersion=1] int32 b; [MinVersion=2] bool c; };') + + def testNewStructFieldNullableReference(self): + """Adding a new nullable reference-typed field is fine if versioned + properly.""" + self.assertBackwardCompatible( + 'struct S { int32 a; };', + 'struct S { int32 a; [MinVersion=1] string? b; };') + + def testStructFieldRename(self): + """Renaming a field has no effect on backward-compatibility.""" + self.assertBackwardCompatible('struct S { int32 x; bool b; };', + 'struct S { int32 a; bool b; };') + + def testStructFieldReorderWithExplicitOrdinals(self): + """Reordering fields has no effect on backward-compatibility when field + ordinals are explicitly labeled and remain unchanged.""" + self.assertBackwardCompatible('struct S { bool b@1; int32 a@0; };', + 'struct S { int32 a@0; bool b@1; };') + + def testNewUnionFieldUnversioned(self): + """Adding a new field to a union without a new (i.e. higher than any + existing version) [MinVersion] tag breaks backward-compatibility.""" + self.assertNotBackwardCompatible('union U { string a; };', + 'union U { string a; string b; };') + + def testUnionFieldRemoval(self): + """Removing a field from a union breaks backward-compatibility.""" + self.assertNotBackwardCompatible('union U { string a; string b; };', + 'union U { string a; };') + + def testUnionFieldTypeChange(self): + """Changing the type of an existing field always breaks + backward-compatibility.""" + self.assertNotBackwardCompatible('union U { string a; };', + 'union U { array a; };') + + def testUnionFieldBecomingOptional(self): + """Changing a field from non-optional to optional breaks + backward-compatibility.""" + self.assertNotBackwardCompatible('union U { string a; };', + 'union U { string? a; };') + + def testFieldNestedTypeChanged(self): + """Changing the definition of a nested type within a field (such as an array + element or interface endpoint type) should only break backward-compatibility + if the changes to that type are not backward-compatible.""" + self.assertBackwardCompatible( + """\ + struct S { string a; }; + struct T { array ss; }; + """, """\ + struct S { + string a; + [MinVersion=1] string? b; + }; + struct T { array ss; }; + """) + self.assertBackwardCompatible( + """\ + interface F { Do(); }; + struct S { pending_receiver r; }; + """, """\ + interface F { + Do(); + [MinVersion=1] Say(); + }; + struct S { pending_receiver r; }; + """) + + def testRecursiveTypeChange(self): + """Recursive types do not break the compatibility checker.""" + self.assertBackwardCompatible( + """\ + struct S { + string a; + array others; + };""", """\ + struct S { + string a; + array others; + [MinVersion=1] string? b; + };""") + + def testUnionFieldBecomingNonOptional(self): + """Changing a field from optional to non-optional breaks + backward-compatibility.""" + self.assertNotBackwardCompatible('union U { string? a; };', + 'union U { string a; };') + + def testUnionFieldOrderChange(self): + """Changing the order of fields breaks backward-compatibility.""" + self.assertNotBackwardCompatible('union U { string a; bool b; };', + 'union U { bool b; string a; };') + self.assertNotBackwardCompatible('union U { string a@0; bool b@1; };', + 'union U { string a@1; bool b@0; };') + + def testUnionFieldMinVersionChange(self): + """Changing the MinVersion of a field breaks backward-compatibility.""" + self.assertNotBackwardCompatible( + 'union U { string a; [MinVersion=1] string b; };', + 'union U { string a; [MinVersion=2] string b; };') + + def testUnionFieldTypeChange(self): + """If a union field's own type definition changes, the containing union + is backward-compatible if and only if the field type's change is + backward-compatible.""" + self.assertBackwardCompatible( + 'struct S {}; union U { S s; };', + 'struct S { [MinVersion=1] int32 x; }; union U { S s; };') + self.assertBackwardCompatible( + '[Extensible] enum E { [Default] kA }; union U { E e; };', + """[Extensible] enum E { + [Default] kA, + [MinVersion=1] kB }; + union U { E e; };""") + self.assertNotBackwardCompatible( + 'struct S {}; union U { S s; };', + 'struct S { int32 x; }; union U { S s; };') + self.assertNotBackwardCompatible( + '[Extensible] enum E { [Default] kA }; union U { E e; };', + '[Extensible] enum E { [Default] kA, kB }; union U { E e; };') + + def testNewUnionFieldWithInvalidMinVersion(self): + """Adding a new field using an existing MinVersion breaks backward- + compatibility.""" + self.assertNotBackwardCompatible( + """\ + union U { + string a; + [MinVersion=1] string b; + }; + """, """\ + union U { + string a; + [MinVersion=1] string b; + [MinVersion=1] string c; + };""") + + def testNewUnionFieldWithValidMinVersion(self): + """Adding a new field is safe if tagged with a MinVersion greater than any + previously used MinVersion in the union.""" + self.assertBackwardCompatible( + 'union U { int32 a; };', + 'union U { int32 a; [MinVersion=1] int32 b; };') + self.assertBackwardCompatible( + 'union U { int32 a; [MinVersion=1] int32 b; };', + 'union U { int32 a; [MinVersion=1] int32 b; [MinVersion=2] bool c; };') + + def testUnionFieldRename(self): + """Renaming a field has no effect on backward-compatibility.""" + self.assertBackwardCompatible('union U { int32 x; bool b; };', + 'union U { int32 a; bool b; };') + + def testUnionFieldReorderWithExplicitOrdinals(self): + """Reordering fields has no effect on backward-compatibility when field + ordinals are explicitly labeled and remain unchanged.""" + self.assertBackwardCompatible('union U { bool b@1; int32 a@0; };', + 'union U { int32 a@0; bool b@1; };') + + def testNewInterfaceMethodUnversioned(self): + """Adding a new method to an interface without a new (i.e. higher than any + existing version) [MinVersion] tag breaks backward-compatibility.""" + self.assertNotBackwardCompatible('interface F { A(); };', + 'interface F { A(); B(); };') + + def testInterfaceMethodRemoval(self): + """Removing a method from an interface breaks backward-compatibility.""" + self.assertNotBackwardCompatible('interface F { A(); B(); };', + 'interface F { A(); };') + + def testInterfaceMethodParamsChanged(self): + """Changes to the parameter list are only backward-compatible if they meet + backward-compatibility requirements of an equivalent struct definition.""" + self.assertNotBackwardCompatible('interface F { A(); };', + 'interface F { A(int32 x); };') + self.assertNotBackwardCompatible('interface F { A(int32 x); };', + 'interface F { A(bool x); };') + self.assertNotBackwardCompatible( + 'interface F { A(int32 x, [MinVersion=1] string? s); };', """\ + interface F { + A(int32 x, [MinVersion=1] string? s, [MinVersion=1] int32 y); + };""") + + self.assertBackwardCompatible('interface F { A(int32 x); };', + 'interface F { A(int32 a); };') + self.assertBackwardCompatible( + 'interface F { A(int32 x); };', + 'interface F { A(int32 x, [MinVersion=1] string? s); };') + + self.assertBackwardCompatible( + 'struct S {}; interface F { A(S s); };', + 'struct S { [MinVersion=1] int32 x; }; interface F { A(S s); };') + self.assertBackwardCompatible( + 'struct S {}; struct T {}; interface F { A(S s); };', + 'struct S {}; struct T {}; interface F { A(T s); };') + self.assertNotBackwardCompatible( + 'struct S {}; struct T { int32 x; }; interface F { A(S s); };', + 'struct S {}; struct T { int32 x; }; interface F { A(T t); };') + + def testInterfaceMethodReplyAdded(self): + """Adding a reply to a message breaks backward-compatibilty.""" + self.assertNotBackwardCompatible('interface F { A(); };', + 'interface F { A() => (); };') + + def testInterfaceMethodReplyRemoved(self): + """Removing a reply from a message breaks backward-compatibility.""" + self.assertNotBackwardCompatible('interface F { A() => (); };', + 'interface F { A(); };') + + def testInterfaceMethodReplyParamsChanged(self): + """Similar to request parameters, a change to reply parameters is considered + backward-compatible if it meets the same backward-compatibility + requirements imposed on equivalent struct changes.""" + self.assertNotBackwardCompatible('interface F { A() => (); };', + 'interface F { A() => (int32 x); };') + self.assertNotBackwardCompatible('interface F { A() => (int32 x); };', + 'interface F { A() => (); };') + self.assertNotBackwardCompatible('interface F { A() => (bool x); };', + 'interface F { A() => (int32 x); };') + + self.assertBackwardCompatible('interface F { A() => (int32 a); };', + 'interface F { A() => (int32 x); };') + self.assertBackwardCompatible( + 'interface F { A() => (int32 x); };', + 'interface F { A() => (int32 x, [MinVersion] string? s); };') + + def testNewInterfaceMethodWithInvalidMinVersion(self): + """Adding a new method to an existing version is not backward-compatible.""" + self.assertNotBackwardCompatible( + """\ + interface F { + A(); + [MinVersion=1] B(); + }; + """, """\ + interface F { + A(); + [MinVersion=1] B(); + [MinVersion=1] C(); + }; + """) + + def testNewInterfaceMethodWithValidMinVersion(self): + """Adding a new method is fine as long as its MinVersion exceeds that of any + method on the old interface definition.""" + self.assertBackwardCompatible('interface F { A(); };', + 'interface F { A(); [MinVersion=1] B(); };') -- cgit v1.2.1