Bug 2038789 - Add extra_deps to GeneratedFile for hidden runtime prereqs, and use it for application.ini r=firefox-build-system-reviewers,sergesanspaille

Some `GeneratedFile` scripts open additional files at runtime
(e.g. #include @TOPOBJDIR@/...), so backends need a way to
declare those as prerequisites without passing them as
positional args.

Also normalizes the `_target_per_file` key to an absolute
path so `GENERATED_FILES` dependencies that cross directory
boundaries actually resolve.

Differential Revision: https://phabricator.services.mozilla.com/D302161

Makefile.in

@@ -24,8 +24,6 @@ endif
 endif
 source-repo.h: $(MDDEPDIR)/source-repo.h.stub
 buildid.h: $(MDDEPDIR)/buildid.h.stub
-# Add explicit dependencies that moz.build can't declare yet.
-build/$(MDDEPDIR)/application.ini.stub: source-repo.h buildid.h
 
 BUILD_BACKEND_FILES := $(addprefix backend.,$(addsuffix Backend,$(BUILD_BACKENDS)))
 

build/moz.build

@@ -101,6 +101,11 @@ if CONFIG["MOZ_APP_BASENAME"]:
         script="../python/mozbuild/mozbuild/action/preprocessor.py",
         entry_point="generate",
         inputs=["application.ini.in"],
+        # `application.ini.in` does `#include @TOPOBJDIR@/source-repo.h`
+        # and `#include @TOPOBJDIR@/buildid.h`. The preprocessor opens
+        # those files at runtime. Declare them as extra_deps so backends
+        # wire the order without passing them as preprocessor inputs.
+        extra_deps=["!/source-repo.h", "!/buildid.h"],
         flags=[
             "-D%s=%s" % (k, "1" if v is True else v)
             for k, v in sorted(appini_defines.items(), key=lambda t: t[0])

build/templates.mozbuild

@@ -248,6 +248,7 @@ def GeneratedFile(name, *names, **kwargs):
     script = kwargs.pop("script", None)
     entry_point = kwargs.pop("entry_point", None)
     inputs = kwargs.pop("inputs", [])
+    extra_deps = kwargs.pop("extra_deps", [])
     flags = kwargs.pop("flags", [])
     force = kwargs.pop("force", False)
     if kwargs:
@@ -269,6 +270,7 @@ def GeneratedFile(name, *names, **kwargs):
     if script and entry_point:
         generated_file.script = script + ":" + entry_point
     generated_file.inputs = inputs
+    generated_file.extra_deps = extra_deps
     generated_file.flags = flags
     generated_file.force = force
 

config/faster/rules.mk

@@ -87,9 +87,3 @@ $(addprefix install-,$(INSTALL_MANIFESTS)): install-%: $(addprefix $(TOPOBJDIR)/
 		-DAB_CD=en-US \
 		$(ACDEFINES) \
 		install_$(subst /,_,$*)
-
-# ============================================================================
-# Below is a set of additional dependencies and variables used to build things
-# that are not supported by data in moz.build.
-
-$(TOPOBJDIR)/build/.deps/application.ini.stub: $(TOPOBJDIR)/buildid.h $(TOPOBJDIR)/source-repo.h

python/mozbuild/mozbuild/backend/make.py

@@ -69,6 +69,14 @@ class MakeBackend(CommonBackend):
         else:
             inputs = []
 
+        # extra_deps are make prerequisites only (not passed to the script as
+        # positional args). Use this for dependencies the recipe needs to
+        # exist on disk but that the script discovers itself at run time
+        # (e.g. preprocessor `#include @TOPOBJDIR@/foo.h`).
+        extra_deps = [
+            self._format_generated_file_input_name(d, obj) for d in obj.extra_deps
+        ]
+
         force = ""
         if obj.force:
             force = " FORCE"
@@ -104,7 +112,7 @@ class MakeBackend(CommonBackend):
 
             ret.append(
                 (
-                    """{stub}: {script}{inputs}{backend}{force}
+                    """{stub}: {script}{inputs}{extra_deps}{backend}{force}
 \t$(REPORT_BUILD)
 \t$(call py_action,file_generate {output},{locale}{script} """  # wrap for E501
                     """{method} {output} {dep_file} {stub}{inputs}{flags})
@@ -115,6 +123,7 @@ class MakeBackend(CommonBackend):
                     output=first_output,
                     dep_file=dep_file,
                     inputs=" " + " ".join(inputs) if inputs else "",
+                    extra_deps=" " + " ".join(extra_deps) if extra_deps else "",
                     flags=(
                         " " + " ".join(shell_quote(f) for f in obj.flags)
                         if obj.flags

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -550,15 +550,15 @@ class RecursiveMakeBackend(MakeBackend):
             reldir = mozpath.relpath(obj.objdir, backend_file.objdir)
             if not reldir:
                 for out in obj.outputs:
-                    target = mozpath.join(obj.relobjdir, out)
+                    target = mozpath.join(obj.objdir, out)
                     assert target not in self._target_per_file
                     self._target_per_file[target] = (obj.relobjdir, tier)
-                for input in obj.inputs:
-                    if isinstance(input, ObjDirPath):
+                for dep in chain(obj.inputs, obj.extra_deps):
+                    if isinstance(dep, ObjDirPath):
                         self._post_process_dependencies.append((
                             obj.relobjdir,
                             tier,
-                            input,
+                            dep,
                         ))
             # For generated files that we handle in the top-level backend file,
             # we want to have a `directory/tier` target depending on the file.

python/mozbuild/mozbuild/frontend/context.py

@@ -1197,6 +1197,7 @@ SchedulingComponents = ContextDerivedTypedRecord(
 GeneratedFilesList = StrictOrderingOnAppendListWithFlagsFactory({
     "script": str,
     "inputs": list,
+    "extra_deps": list,
     "force": bool,
     "flags": list,
 })
@@ -1521,13 +1522,14 @@ VARIABLES = {
         Unless you have a reason not to, use the GeneratedFile template rather
         than referencing GENERATED_FILES directly. The GeneratedFile template
         has all the same arguments as the attributes listed below (``script``,
-        ``inputs``, ``flags``, ``force``), plus an additional ``entry_point``
-        argument to specify a particular function to run in the given script.
+        ``inputs``, ``extra_deps``, ``flags``, ``force``), plus an additional
+        ``entry_point`` argument to specify a particular function to run in
+        the given script.
 
         This variable contains a list of files for the build system to
         generate at export time. The generation method may be declared
-        with optional ``script``, ``inputs``, ``flags``, and ``force``
-        attributes on individual entries.
+        with optional ``script``, ``inputs``, ``extra_deps``, ``flags``,
+        and ``force`` attributes on individual entries.
         If the optional ``script`` attribute is not present on an entry, it
         is assumed that rules for generating the file are present in
         the associated Makefile.in.
@@ -1566,6 +1568,16 @@ VARIABLES = {
         When the ``flags`` attribute is present, the given list of flags is
         passed as extra arguments following the inputs.
 
+        When the ``extra_deps`` attribute is present, the listed paths are
+        added as build-graph prerequisites for the generation step but are
+        not passed to ``script`` as positional arguments. Use this when the
+        script opens additional files itself at runtime (e.g. via the
+        preprocessor's #include @TOPOBJDIR@/...) and those files must
+        therefore exist on disk before the step runs. An objdir-relative
+        path like ``"!/source-repo.h"`` resolves against ``$topobjdir``,
+        and a plain path resolves relative to the directory containing the
+        moz.build file.
+
         When the ``force`` attribute is present, the file is generated every
         build, regardless of whether it is stale.  This is special to the
         RecursiveMake backend and intended for special situations only (e.g.,

python/mozbuild/mozbuild/frontend/data.py

@@ -1355,6 +1355,7 @@ class GeneratedFile(ContextDerived):
         "method",
         "outputs",
         "inputs",
+        "extra_deps",
         "flags",
         "required_before_export",
         "required_before_compile",
@@ -1374,6 +1375,7 @@ class GeneratedFile(ContextDerived):
         localized=False,
         force=False,
         required_during_compile=None,
+        extra_deps=(),
     ):
         ContextDerived.__init__(self, context)
         self.script = script
@@ -1381,6 +1383,7 @@ class GeneratedFile(ContextDerived):
         self.outputs = outputs if isinstance(outputs, tuple) else (outputs,)
         self.inputs = inputs
         self.flags = flags
+        self.extra_deps = extra_deps
         self.localized = localized
         self.force = force
 

python/mozbuild/mozbuild/frontend/emitter.py

@@ -1759,6 +1759,16 @@ class TreeMetadataEmitter(LoggingMixin):
                         )
                     inputs.append(p)
 
+                extra_deps = []
+                for d in flags.extra_deps:
+                    p = Path(context, d)
+                    if isinstance(p, SourcePath) and not os.path.exists(p.full_path):
+                        raise SandboxValidationError(
+                            f"extra_dep for generating {f} does not exist: {p.full_path}",
+                            context,
+                        )
+                    extra_deps.append(p)
+
                 yield GeneratedFile(
                     context,
                     script,
@@ -1768,6 +1778,7 @@ class TreeMetadataEmitter(LoggingMixin):
                     flags.flags,
                     localized=localized,
                     force=flags.force,
+                    extra_deps=extra_deps,
                 )
 
     def _process_test_manifests(self, context):

python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/moz.build

@@ -0,0 +1,9 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+GENERATED_FILES += ["consumer.c"]
+
+consumer = GENERATED_FILES["consumer.c"]
+consumer.script = "generate-consumer.py"
+consumer.inputs = ["input-data"]
+consumer.extra_deps = ["source-extra", "!/producer/producer.c"]

python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/moz.build

@@ -0,0 +1,7 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+DIRS += [
+    "producer",
+    "consumer",
+]

python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/producer/moz.build

@@ -0,0 +1,7 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+GENERATED_FILES += ["producer.c"]
+
+producer = GENERATED_FILES["producer.c"]
+producer.script = "generate-producer.py"

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -449,6 +449,39 @@ class TestRecursiveMakeBackend(BackendTester):
         self.maxDiff = None
         self.assertEqual(lines, expected)
 
+    def test_generated_files_extra_deps(self):
+        """Ensure GENERATED_FILES extra_deps are handled properly."""
+        env = self._consume("generated-files-extra-deps", RecursiveMakeBackend)
+
+        backend_path = mozpath.join(env.topobjdir, "consumer", "backend.mk")
+        lines = [l.strip() for l in open(backend_path).readlines()[2:]]
+
+        expected = [
+            "include $(topsrcdir)/config/AB_rCD.mk",
+            "PRE_COMPILE_TARGETS += $(MDDEPDIR)/consumer.c.stub",
+            "consumer.c: $(MDDEPDIR)/consumer.c.stub ;",
+            "EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/consumer.c.pp",
+            "$(MDDEPDIR)/consumer.c.stub: "
+            f"{env.topsrcdir}/consumer/generate-consumer.py "
+            "$(srcdir)/input-data "
+            "$(srcdir)/source-extra "
+            "$(DEPTH)/producer/producer.c",
+            "$(REPORT_BUILD)",
+            "$(call py_action,file_generate consumer.c,"
+            f"{env.topsrcdir}/consumer/generate-consumer.py main "
+            "consumer.c $(MDDEPDIR)/consumer.c.pp "
+            "$(MDDEPDIR)/consumer.c.stub $(srcdir)/input-data)",
+            "@$(TOUCH) $@",
+            "",
+        ]
+
+        self.maxDiff = None
+        self.assertEqual(lines, expected)
+
+        root_deps_path = mozpath.join(env.topobjdir, "root-deps.mk")
+        lines = [l.strip() for l in open(root_deps_path).readlines()]
+        self.assertIn("consumer/pre-compile: producer/pre-compile", lines)
+
     def test_generated_files_force(self):
         """Ensure GENERATED_FILES with .force is handled properly."""
         env = self._consume("generated-files-force", RecursiveMakeBackend)

python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps-missing/moz.build

@@ -0,0 +1,9 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+GENERATED_FILES += ["foo.c"]
+
+foo = GENERATED_FILES["foo.c"]
+foo.script = "script.py"
+foo.inputs = []
+foo.extra_deps = ["does-not-exist.h"]

python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps/moz.build

@@ -0,0 +1,9 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+GENERATED_FILES += ["foo.c"]
+
+foo = GENERATED_FILES["foo.c"]
+foo.script = "script.py"
+foo.inputs = []
+foo.extra_deps = ["!/generated-header.h"]

python/mozbuild/mozbuild/test/frontend/test_emitter.py

@@ -656,6 +656,29 @@ class TestEmitterBasic(unittest.TestCase):
         ):
             self.read_topsrcdir(reader)
 
+    def test_generated_files_extra_deps(self):
+        """Test that GENERATED_FILES extra_deps is properly parsed."""
+        reader = self.reader("generated-files-extra-deps")
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 1)
+        o = objs[0]
+        self.assertIsInstance(o, GeneratedFile)
+        self.assertEqual(o.outputs, ("foo.c",))
+        self.assertEqual(o.inputs, [])
+        self.assertEqual(len(o.extra_deps), 1)
+        self.assertIsInstance(o.extra_deps[0], ObjDirPath)
+        self.assertEqual(o.extra_deps[0], "!/generated-header.h")
+
+    def test_generated_files_extra_deps_missing(self):
+        """Test that a missing extra_deps source is an error."""
+        reader = self.reader("generated-files-extra-deps-missing")
+        with self.assertRaisesRegex(
+            SandboxValidationError,
+            "extra_dep for generating foo.c does not exist",
+        ):
+            self.read_topsrcdir(reader)
+
     def test_exports(self):
         reader = self.reader("exports")
         objs = self.read_topsrcdir(reader)