meson: Move all code generation scripts to utils/codegen/

We have multiple code generation scripts in utils/, mixed with other
miscellaneous utilities, as well as a larger code base based on mojom in
utils/ipc/. To make code sharing easier between the generator scripts,
without creating a mess in the utils/ directory, move all the code
generation code to utils/codegen/.

Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Reviewed-by: Daniel Scally <dan.scally@ideasonboard.com>
Reviewed-by: Paul Elder <paul.elder@ideasonboard.com>

1 files changed, 502 insertions, 0 deletions

diff --git a/utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser.py b/utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser.py
new file mode 100755
index 00000000..9693090e
--- /dev/null
+++ b/utils/codegen/ipc/mojo/public/tools/mojom/mojom_parser.py
@@ -0,0 +1,502 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Parses mojom IDL files.
+
+This script parses one or more input mojom files and produces corresponding
+module files fully describing the definitions contained within each mojom. The
+module data is pickled and can be easily consumed by other tools to, e.g.,
+generate usable language bindings.
+"""
+
+import argparse
+import builtins
+import codecs
+import errno
+import json
+import logging
+import multiprocessing
+import os
+import os.path
+import sys
+import traceback
+from collections import defaultdict
+
+from mojom.generate import module
+from mojom.generate import translate
+from mojom.parse import parser
+from mojom.parse import conditional_features
+
+
+# Disable this for easier debugging.
+_ENABLE_MULTIPROCESSING = True
+
+# https://docs.python.org/3/library/multiprocessing.html#:~:text=bpo-33725
+if __name__ == '__main__' and sys.platform == 'darwin':
+  multiprocessing.set_start_method('fork')
+_MULTIPROCESSING_USES_FORK = multiprocessing.get_start_method() == 'fork'
+
+
+def _ResolveRelativeImportPath(path, roots):
+  """Attempts to resolve a relative import path against a set of possible roots.
+
+  Args:
+    path: The relative import path to resolve.
+    roots: A list of absolute paths which will be checked in descending length
+        order for a match against path.
+
+  Returns:
+    A normalized absolute path combining one of the roots with the input path if
+    and only if such a file exists.
+
+  Raises:
+    ValueError: The path could not be resolved against any of the given roots.
+  """
+  for root in reversed(sorted(roots, key=len)):
+    abs_path = os.path.join(root, path)
+    if os.path.isfile(abs_path):
+      return os.path.normcase(os.path.normpath(abs_path))
+
+  raise ValueError('"%s" does not exist in any of %s' % (path, roots))
+
+
+def RebaseAbsolutePath(path, roots):
+  """Rewrites an absolute file path as relative to an absolute directory path in
+  roots.
+
+  Args:
+    path: The absolute path of an existing file.
+    roots: A list of absolute directory paths. The given path argument must fall
+        within one of these directories.
+
+  Returns:
+    A path equivalent to the input path, but relative to one of the provided
+    roots. If the input path falls within multiple roots, the longest root is
+    chosen (and thus the shortest relative path is returned).
+
+    Paths returned by this method always use forward slashes as a separator to
+    mirror mojom import syntax.
+
+  Raises:
+    ValueError if the given path does not fall within any of the listed roots.
+  """
+  assert os.path.isabs(path)
+  assert os.path.isfile(path)
+  assert all(map(os.path.isabs, roots))
+
+  sorted_roots = list(reversed(sorted(roots, key=len)))
+
+  def try_rebase_path(path, root):
+    head, rebased_path = os.path.split(path)
+    while head != root:
+      head, tail = os.path.split(head)
+      if not tail:
+        return None
+      rebased_path = os.path.join(tail, rebased_path)
+    return rebased_path
+
+  for root in sorted_roots:
+    relative_path = try_rebase_path(path, root)
+    if relative_path:
+      # TODO(crbug.com/953884): Use pathlib for this kind of thing once we're
+      # fully migrated to Python 3.
+      return relative_path.replace('\\', '/')
+
+  raise ValueError('%s does not fall within any of %s' % (path, sorted_roots))
+
+
+def _GetModuleFilename(mojom_filename):
+  return mojom_filename + '-module'
+
+
+def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts,
+                       dependencies, loaded_modules, module_metadata):
+  """Recursively ensures that a module and its dependencies are loaded.
+
+  Args:
+    mojom_abspath: An absolute file path pointing to a mojom file to load.
+    module_path: The relative path used to identify mojom_abspath.
+    abs_paths: A mapping from module paths to absolute file paths for all
+        inputs given to this execution of the script.
+    asts: A map from each input mojom's absolute path to its parsed AST.
+    dependencies: A mapping of which input mojoms depend on each other, indexed
+        by absolute file path.
+    loaded_modules: A mapping of all modules loaded so far, including non-input
+        modules that were pulled in as transitive dependencies of the inputs.
+    module_metadata: Metadata to be attached to every module loaded by this
+        helper.
+
+  Returns:
+    None
+
+    On return, loaded_modules will be populated with the loaded input mojom's
+    Module as well as the Modules of all of its transitive dependencies."""
+
+  if mojom_abspath in loaded_modules:
+    # Already done.
+    return
+
+  for dep_abspath, dep_path in sorted(dependencies[mojom_abspath]):
+    if dep_abspath not in loaded_modules:
+      _EnsureInputLoaded(dep_abspath, dep_path, abs_paths, asts, dependencies,
+                         loaded_modules, module_metadata)
+
+  imports = {}
+  for imp in asts[mojom_abspath].import_list:
+    path = imp.import_filename
+    imports[path] = loaded_modules[abs_paths[path]]
+  loaded_modules[mojom_abspath] = translate.OrderedModule(
+      asts[mojom_abspath], module_path, imports)
+  loaded_modules[mojom_abspath].metadata = dict(module_metadata)
+
+
+def _CollectAllowedImportsFromBuildMetadata(build_metadata_filename):
+  allowed_imports = set()
+  processed_deps = set()
+
+  def collect(metadata_filename):
+    processed_deps.add(metadata_filename)
+
+    # Paths in the metadata file are relative to the metadata file's dir.
+    metadata_dir = os.path.abspath(os.path.dirname(metadata_filename))
+
+    def to_abs(s):
+      return os.path.normpath(os.path.join(metadata_dir, s))
+
+    with open(metadata_filename) as f:
+      metadata = json.load(f)
+      allowed_imports.update(
+          [os.path.normcase(to_abs(s)) for s in metadata['sources']])
+      for dep_metadata in metadata['deps']:
+        dep_metadata = to_abs(dep_metadata)
+        if dep_metadata not in processed_deps:
+          collect(dep_metadata)
+
+  collect(build_metadata_filename)
+  return allowed_imports
+
+
+# multiprocessing helper.
+def _ParseAstHelper(mojom_abspath, enabled_features):
+  with codecs.open(mojom_abspath, encoding='utf-8') as f:
+    ast = parser.Parse(f.read(), mojom_abspath)
+    conditional_features.RemoveDisabledDefinitions(ast, enabled_features)
+    return mojom_abspath, ast
+
+
+# multiprocessing helper.
+def _SerializeHelper(mojom_abspath, mojom_path):
+  module_path = os.path.join(_SerializeHelper.output_root_path,
+                             _GetModuleFilename(mojom_path))
+  module_dir = os.path.dirname(module_path)
+  if not os.path.exists(module_dir):
+    try:
+      # Python 2 doesn't support exist_ok on makedirs(), so we just ignore
+      # that failure if it happens. It's possible during build due to races
+      # among build steps with module outputs in the same directory.
+      os.makedirs(module_dir)
+    except OSError as e:
+      if e.errno != errno.EEXIST:
+        raise
+  with open(module_path, 'wb') as f:
+    _SerializeHelper.loaded_modules[mojom_abspath].Dump(f)
+
+
+class _ExceptionWrapper:
+  def __init__(self):
+    # Do not capture exception object to ensure pickling works.
+    self.formatted_trace = traceback.format_exc()
+
+
+class _FuncWrapper:
+  """Marshals exceptions and spreads args."""
+
+  def __init__(self, func):
+    self._func = func
+
+  def __call__(self, args):
+    # multiprocessing does not gracefully handle excptions.
+    # https://crbug.com/1219044
+    try:
+      return self._func(*args)
+    except:  # pylint: disable=bare-except
+      return _ExceptionWrapper()
+
+
+def _Shard(target_func, arg_list, processes=None):
+  arg_list = list(arg_list)
+  if processes is None:
+    processes = multiprocessing.cpu_count()
+  # Seems optimal to have each process perform at least 2 tasks.
+  processes = min(processes, len(arg_list) // 2)
+
+  if sys.platform == 'win32':
+    # TODO(crbug.com/1190269) - we can't use more than 56
+    # cores on Windows or Python3 may hang.
+    processes = min(processes, 56)
+
+  # Don't spin up processes unless there is enough work to merit doing so.
+  if not _ENABLE_MULTIPROCESSING or processes < 2:
+    for arg_tuple in arg_list:
+      yield target_func(*arg_tuple)
+    return
+
+  pool = multiprocessing.Pool(processes=processes)
+  try:
+    wrapped_func = _FuncWrapper(target_func)
+    for result in pool.imap_unordered(wrapped_func, arg_list):
+      if isinstance(result, _ExceptionWrapper):
+        sys.stderr.write(result.formatted_trace)
+        sys.exit(1)
+      yield result
+  finally:
+    pool.close()
+    pool.join()  # Needed on Windows to avoid WindowsError during terminate.
+    pool.terminate()
+
+
+def _ParseMojoms(mojom_files,
+                 input_root_paths,
+                 output_root_path,
+                 module_root_paths,
+                 enabled_features,
+                 module_metadata,
+                 allowed_imports=None):
+  """Parses a set of mojom files and produces serialized module outputs.
+
+  Args:
+    mojom_files: A list of mojom files to process. Paths must be absolute paths
+        which fall within one of the input or output root paths.
+    input_root_paths: A list of absolute filesystem paths which may be used to
+        resolve relative mojom file paths.
+    output_root_path: An absolute filesystem path which will service as the root
+        for all emitted artifacts. Artifacts produced from a given mojom file
+        are based on the mojom's relative path, rebased onto this path.
+        Additionally, the script expects this root to contain already-generated
+        modules for any transitive dependencies not listed in mojom_files.
+    module_root_paths: A list of absolute filesystem paths which contain
+        already-generated modules for any non-transitive dependencies.
+    enabled_features: A list of enabled feature names, controlling which AST
+        nodes are filtered by [EnableIf] or [EnableIfNot] attributes.
+    module_metadata: A list of 2-tuples representing metadata key-value pairs to
+        attach to each compiled module output.
+
+  Returns:
+    None.
+
+    Upon completion, a mojom-module file will be saved for each input mojom.
+  """
+  assert input_root_paths
+  assert output_root_path
+
+  loaded_mojom_asts = {}
+  loaded_modules = {}
+  input_dependencies = defaultdict(set)
+  mojom_files_to_parse = dict((os.path.normcase(abs_path),
+                               RebaseAbsolutePath(abs_path, input_root_paths))
+                              for abs_path in mojom_files)
+  abs_paths = dict(
+      (path, abs_path) for abs_path, path in mojom_files_to_parse.items())
+
+  logging.info('Parsing %d .mojom into ASTs', len(mojom_files_to_parse))
+  map_args = ((mojom_abspath, enabled_features)
+              for mojom_abspath in mojom_files_to_parse)
+  for mojom_abspath, ast in _Shard(_ParseAstHelper, map_args):
+    loaded_mojom_asts[mojom_abspath] = ast
+
+  logging.info('Processing dependencies')
+  for mojom_abspath, ast in sorted(loaded_mojom_asts.items()):
+    invalid_imports = []
+    for imp in ast.import_list:
+      import_abspath = _ResolveRelativeImportPath(imp.import_filename,
+                                                  input_root_paths)
+      if allowed_imports and import_abspath not in allowed_imports:
+        invalid_imports.append(imp.import_filename)
+
+      abs_paths[imp.import_filename] = import_abspath
+      if import_abspath in mojom_files_to_parse:
+        # This import is in the input list, so we're going to translate it
+        # into a module below; however it's also a dependency of another input
+        # module. We retain record of dependencies to help with input
+        # processing later.
+        input_dependencies[mojom_abspath].add(
+            (import_abspath, imp.import_filename))
+      elif import_abspath not in loaded_modules:
+        # We have an import that isn't being parsed right now. It must already
+        # be parsed and have a module file sitting in a corresponding output
+        # location.
+        module_path = _GetModuleFilename(imp.import_filename)
+        module_abspath = _ResolveRelativeImportPath(
+            module_path, module_root_paths + [output_root_path])
+        with open(module_abspath, 'rb') as module_file:
+          loaded_modules[import_abspath] = module.Module.Load(module_file)
+
+    if invalid_imports:
+      raise ValueError(
+          '\nThe file %s imports the following files not allowed by build '
+          'dependencies:\n\n%s\n' % (mojom_abspath, '\n'.join(invalid_imports)))
+  logging.info('Loaded %d modules from dependencies', len(loaded_modules))
+
+  # At this point all transitive imports not listed as inputs have been loaded
+  # and we have a complete dependency tree of the unprocessed inputs. Now we can
+  # load all the inputs, resolving dependencies among them recursively as we go.
+  logging.info('Ensuring inputs are loaded')
+  num_existing_modules_loaded = len(loaded_modules)
+  for mojom_abspath, mojom_path in mojom_files_to_parse.items():
+    _EnsureInputLoaded(mojom_abspath, mojom_path, abs_paths, loaded_mojom_asts,
+                       input_dependencies, loaded_modules, module_metadata)
+  assert (num_existing_modules_loaded +
+          len(mojom_files_to_parse) == len(loaded_modules))
+
+  # Now we have fully translated modules for every input and every transitive
+  # dependency. We can dump the modules to disk for other tools to use.
+  logging.info('Serializing %d modules', len(mojom_files_to_parse))
+
+  # Windows does not use fork() for multiprocessing, so we'd need to pass
+  # loaded_module via IPC rather than via globals. Doing so is slower than not
+  # using multiprocessing.
+  _SerializeHelper.loaded_modules = loaded_modules
+  _SerializeHelper.output_root_path = output_root_path
+  # Doesn't seem to help past 4. Perhaps IO bound here?
+  processes = 4 if _MULTIPROCESSING_USES_FORK else 0
+  map_args = mojom_files_to_parse.items()
+  for _ in _Shard(_SerializeHelper, map_args, processes=processes):
+    pass
+
+
+def Run(command_line):
+  debug_logging = os.environ.get('MOJOM_PARSER_DEBUG', '0') != '0'
+  logging.basicConfig(level=logging.DEBUG if debug_logging else logging.WARNING,
+                      format='%(levelname).1s %(relativeCreated)6d %(message)s')
+  logging.info('Started (%s)', os.path.basename(sys.argv[0]))
+
+  arg_parser = argparse.ArgumentParser(
+      description="""
+Parses one or more mojom files and produces corresponding module outputs fully
+describing the definitions therein. The output is exhaustive, stable, and
+sufficient for another tool to consume and emit e.g. usable language
+bindings based on the original mojoms.""",
+      epilog="""
+Note that each transitive import dependency reachable from the input mojoms must
+either also be listed as an input or must have its corresponding compiled module
+already present in the provided output root.""")
+
+  arg_parser.add_argument(
+      '--input-root',
+      default=[],
+      action='append',
+      metavar='ROOT',
+      dest='input_root_paths',
+      help='Adds ROOT to the set of root paths against which relative input '
+      'paths should be resolved. Provided root paths are always searched '
+      'in order from longest absolute path to shortest.')
+  arg_parser.add_argument(
+      '--output-root',
+      action='store',
+      required=True,
+      dest='output_root_path',
+      metavar='ROOT',
+      help='Use ROOT as the root path in which the parser should emit compiled '
+      'modules for each processed input mojom. The path of emitted module is '
+      'based on the relative input path, rebased onto this root. Note that '
+      'ROOT is also searched for existing modules of any transitive imports '
+      'which were not included in the set of inputs.')
+  arg_parser.add_argument(
+      '--module-root',
+      default=[],
+      action='append',
+      metavar='ROOT',
+      dest='module_root_paths',
+      help='Adds ROOT to the set of root paths to search for existing modules '
+      'of non-transitive imports. Provided root paths are always searched in '
+      'order from longest absolute path to shortest.')
+  arg_parser.add_argument(
+      '--mojoms',
+      nargs='+',
+      dest='mojom_files',
+      default=[],
+      metavar='MOJOM_FILE',
+      help='Input mojom filename(s). Each filename must be either an absolute '
+      'path which falls within one of the given input or output roots, or a '
+      'relative path the parser will attempt to resolve using each of those '
+      'roots in unspecified order.')
+  arg_parser.add_argument(
+      '--mojom-file-list',
+      action='store',
+      metavar='LIST_FILENAME',
+      help='Input file whose contents are a list of mojoms to process. This '
+      'may be provided in lieu of --mojoms to avoid hitting command line '
+      'length limtations')
+  arg_parser.add_argument(
+      '--enable-feature',
+      dest='enabled_features',
+      default=[],
+      action='append',
+      metavar='FEATURE',
+      help='Enables a named feature when parsing the given mojoms. Features '
+      'are identified by arbitrary string values. Specifying this flag with a '
+      'given FEATURE name will cause the parser to process any syntax elements '
+      'tagged with an [EnableIf=FEATURE] or [EnableIfNot] attribute. If this '
+      'flag is not provided for a given FEATURE, such tagged elements are '
+      'discarded by the parser and will not be present in the compiled output.')
+  arg_parser.add_argument(
+      '--check-imports',
+      dest='build_metadata_filename',
+      action='store',
+      metavar='METADATA_FILENAME',
+      help='Instructs the parser to check imports against a set of allowed '
+      'imports. Allowed imports are based on build metadata within '
+      'METADATA_FILENAME. This is a JSON file with a `sources` key listing '
+      'paths to the set of input mojom files being processed by this parser '
+      'run, and a `deps` key listing paths to metadata files for any '
+      'dependencies of these inputs. This feature can be used to implement '
+      'build-time dependency checking for mojom imports, where each build '
+      'metadata file corresponds to a build target in the dependency graph of '
+      'a typical build system.')
+  arg_parser.add_argument(
+      '--add-module-metadata',
+      dest='module_metadata',
+      default=[],
+      action='append',
+      metavar='KEY=VALUE',
+      help='Adds a metadata key-value pair to the output module. This can be '
+      'used by build toolchains to augment parsed mojom modules with product-'
+      'specific metadata for later extraction and use by custom bindings '
+      'generators.')
+
+  args, _ = arg_parser.parse_known_args(command_line)
+  if args.mojom_file_list:
+    with open(args.mojom_file_list) as f:
+      args.mojom_files.extend(f.read().split())
+
+  if not args.mojom_files:
+    raise ValueError(
+        'Must list at least one mojom file via --mojoms or --mojom-file-list')
+
+  mojom_files = list(map(os.path.abspath, args.mojom_files))
+  input_roots = list(map(os.path.abspath, args.input_root_paths))
+  output_root = os.path.abspath(args.output_root_path)
+  module_roots = list(map(os.path.abspath, args.module_root_paths))
+
+  if args.build_metadata_filename:
+    allowed_imports = _CollectAllowedImportsFromBuildMetadata(
+        args.build_metadata_filename)
+  else:
+    allowed_imports = None
+
+  module_metadata = list(
+      map(lambda kvp: tuple(kvp.split('=')), args.module_metadata))
+  _ParseMojoms(mojom_files, input_roots, output_root, module_roots,
+               args.enabled_features, module_metadata, allowed_imports)
+  logging.info('Finished')
+
+
+if __name__ == '__main__':
+  Run(sys.argv[1:])
+  # Exit without running GC, which can save multiple seconds due to the large
+  # number of object created. But flush is necessary as os._exit doesn't do
+  # that.
+  sys.stdout.flush()
+  sys.stderr.flush()
+  os._exit(0)