Implement reproducible event generation

.github/workflows/acceptancetest_mg7.yml

@@ -243,6 +243,92 @@ jobs:
             export PATH="$HOME/.cache/HEPtools/bin:$PATH"
             ./tests/test_manager.py test_flavor_grouping_consistency_mg7 -pA -t0 -l INFO
 
+  acceptancetest_mg7_vegas_reproducibility:
+    needs: build_madspace
+    # mg7 seeded-generation reproducibility (p p > t t~, plain VEGAS-optimized
+    # run): the same run_card seed must give a byte-identical LHE file across
+    # independent runs, a different seed must not. Self-skips if the madspace
+    # + LHAPDF(NNPDF23) stack is unavailable.
+    runs-on: ubuntu-24.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.fork == true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/checkout_mg5
+      - uses: ./.github/actions/install_madspace
+      - uses: ./.github/actions/restore-pip-cache
+      - uses: ./.github/actions/restore_heptools
+      - name: test one of the test test_vegas_reproducibility_mg7
+        run: |
+            cd $GITHUB_WORKSPACE
+            export PATH="$HOME/.cache/HEPtools/bin:$PATH"
+            ./tests/test_manager.py test_vegas_reproducibility_mg7 -pA -t0 -l INFO
+
+  acceptancetest_mg7_madnis_reproducible_mode:
+    needs: build_madspace
+    # mg7 madnis training (p p > t t~): training itself must be byte-identical
+    # for the same seed independent of the cpu thread pool size, both with
+    # online-only training and with buffered (off-policy replay) training
+    # enabled. Self-skips if the madspace + LHAPDF(NNPDF23) stack is
+    # unavailable.
+    runs-on: ubuntu-24.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.fork == true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/checkout_mg5
+      - uses: ./.github/actions/install_madspace
+      - uses: ./.github/actions/restore-pip-cache
+      - uses: ./.github/actions/restore_heptools
+      - name: test one of the test test_madnis_reproducible_mode_mg7
+        run: |
+            cd $GITHUB_WORKSPACE
+            export PATH="$HOME/.cache/HEPtools/bin:$PATH"
+            ./tests/test_manager.py test_madnis_reproducible_mode_mg7 -pA -t0 -l INFO
+
+  acceptancetest_mg7_gridpack_reproducibility:
+    needs: build_madspace
+    # mg7 seeded-generation reproducibility for a gridpack trained with
+    # madnis (p p > t t~): the gridpack's own bin/generate_events --seed must
+    # give a byte-identical LHE file across independent runs with the same
+    # seed, and a different result for a different seed. Training itself is
+    # not required to be deterministic. Self-skips if the madspace +
+    # LHAPDF(NNPDF23) stack is unavailable.
+    runs-on: ubuntu-24.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.fork == true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/checkout_mg5
+      - uses: ./.github/actions/install_madspace
+      - uses: ./.github/actions/restore-pip-cache
+      - uses: ./.github/actions/restore_heptools
+      - name: test one of the test test_gridpack_reproducibility_mg7
+        run: |
+            cd $GITHUB_WORKSPACE
+            export PATH="$HOME/.cache/HEPtools/bin:$PATH"
+            ./tests/test_manager.py test_gridpack_reproducibility_mg7 -pA -t0 -l INFO
+
+  acceptancetest_mg7_gridpack_reproducibility_vegas:
+    needs: build_madspace
+    # mg7 seeded-generation reproducibility for a plain VEGAS-optimized
+    # gridpack (p p > t t~, no madnis training): the gridpack's own
+    # bin/generate_events --seed must give a byte-identical LHE file across
+    # independent runs with the same seed, and a different result for a
+    # different seed. Companion to acceptancetest_mg7_gridpack_reproducibility,
+    # which covers the madnis-trained case. Self-skips if the madspace +
+    # LHAPDF(NNPDF23) stack is unavailable.
+    runs-on: ubuntu-24.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.fork == true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/checkout_mg5
+      - uses: ./.github/actions/install_madspace
+      - uses: ./.github/actions/restore-pip-cache
+      - uses: ./.github/actions/restore_heptools
+      - name: test one of the test test_gridpack_reproducibility_vegas_mg7
+        run: |
+            cd $GITHUB_WORKSPACE
+            export PATH="$HOME/.cache/HEPtools/bin:$PATH"
+            ./tests/test_manager.py test_gridpack_reproducibility_vegas_mg7 -pA -t0 -l INFO
+
   acceptancetest_mg7_merged_flavor_uq:
     needs: build_madspace
     # mg7 cross-section for the merged-flavor u q > u q (q = u d), pinned to the

.github/workflows/madspace.yml

@@ -21,18 +21,16 @@ jobs:
               yum -y install openblas-devel yum-utils &&
               yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo &&
               yum-config-manager --add-repo /project/.github/workflows/rocm.repo &&
-              yum -y install cuda-nvcc-12-8 cuda-cudart-devel-12-8 cuda-thrust-12-8 libcublas-devel-12-8 libcurand-devel-12-8 rocm-hip-runtime-devel rocblas-devel rocrand-devel rocthrust-devel hipcub-devel
+              yum -y install cuda-nvcc-12-8 cuda-cudart-devel-12-8 cuda-thrust-12-8 libcublas-devel-12-8 rocm-hip-runtime-devel rocblas-devel rocthrust-devel hipcub-devel
             CIBW_REPAIR_WHEEL_COMMAND: >
               auditwheel repair
               --exclude libmadspace.so
               --exclude 'libcublas.so.*'
-              --exclude 'libcurand.so.*'
               --exclude 'libcublasLt.so.*'
               --exclude 'libcudart.so.*'
               --exclude 'libamdhip64.so.*'
               --exclude 'libhipblaslt.so.*'
               --exclude 'librocblas.so.*'
-              --exclude 'librocrand.so.*'
               --exclude 'libroctx64.so.*'
               -w {dest_dir} {wheel}
             CIBW_ENVIRONMENT: >

madgraph/iolibs/template_files/mg7/gridpack.py

@@ -31,7 +31,6 @@
 import tomllib
 import argparse
 
-
 def resolve_verbosity(verbosity: str) -> str:
     """Resolve the run_card "auto" verbosity to "pretty"/"log" depending on
     whether stdout is attached to a terminal; other values pass through
@@ -41,6 +40,14 @@ def resolve_verbosity(verbosity: str) -> str:
     return verbosity
 
 
+def resolve_seed(seed: int) -> int:
+    """Resolve the run_card "seed": -1 draws a fresh 64-bit seed via
+    os.urandom, any other value is used as-is."""
+    if seed == -1:
+        return int.from_bytes(os.urandom(8), "big")
+    return seed
+
+
 def main() -> None:
     # load run card and metadata. Use the RunCardMG7 representation when the
     # madgraph package is importable; gridpacks are meant to be portable, so
@@ -69,6 +76,12 @@ def main() -> None:
     # parse command line arguments
     parser = argparse.ArgumentParser()
     parser.add_argument("--run_name", type=str, default=run_args["run_name"])
+    parser.add_argument(
+        "--seed", type=int, default=run_args.get("seed", -1),
+        help="every run is reproducible from its seed; -1 draws a fresh random "
+             "seed each run instead of fixing one here (still recorded in the "
+             "run's info.json)"
+    )
     parser.add_argument("--device", type=str, nargs="*")
     parser.add_argument(
         "--cpu_thread_pool_size", type=int, default=run_args["cpu_thread_pool_size"]
@@ -94,6 +107,7 @@ def main() -> None:
     parser.add_argument("--cpu_batch_size", type=int, default=gen_args["cpu_batch_size"])
     parser.add_argument("--gpu_batch_size", type=int, default=gen_args["gpu_batch_size"])
     args = parser.parse_args()
+    seed = resolve_seed(args.seed)
 
     # initialize event directory
     run_name = args.run_name
@@ -173,6 +187,7 @@ def main() -> None:
         channels=channel_generators,
         status_file=ms.StatusFile(os.path.join(run_path, "info.json")),
         config=config,
+        seed=seed,
     )
 
     # run generation

madgraph/iolibs/template_files/mg7/madevent.py

@@ -115,6 +115,14 @@ def resolve_verbosity(verbosity: str) -> str:
     return verbosity
 
 
+def resolve_seed(seed: int) -> int:
+    """Resolve the run_card "seed": -1 draws a fresh 64-bit seed via
+    os.urandom, any other value is used as-is."""
+    if seed == -1:
+        return int.from_bytes(os.urandom(8), "big")
+    return seed
+
+
 def resolve_cppauto_backend(build_path: str) -> str:
     """Ask the matrix-element Makefile to resolve ``cppauto``.
 
@@ -230,6 +238,10 @@ def __init__(self):
 
     def load_cards(self) -> None:
         self.run_card = RunCardMG7(os.path.join("Cards", "run_card.toml"))
+        # Resolved once so every generator built during this run shares the same
+        # seed; the concrete value (even if randomly drawn) is recorded in each
+        # generator's info.json.
+        self.run_seed = resolve_seed(self.run_card["run"]["seed"])
         self.param_card_path = os.path.join("Cards", "param_card.dat")
         self.param_card = ParamCard(self.param_card_path)
         with open(os.path.join("SubProcesses", "subprocesses.json")) as f:
@@ -269,7 +281,7 @@ def init_context(self) -> None:
         self.device_types = []
         self.devices = []
         self.pool_sizes = []
-        for i, device_name in enumerate(device_names):
+        for device_name in device_names:
             if ":" in device_name:
                 device_type, device_index_str = device_name.split(":")
                 device_index = int(device_index_str)
@@ -566,6 +578,7 @@ def build_event_generator(self, phasespaces: list[PhaseSpace]) -> ms.EventGenera
             channels=channel_generators,
             status_file=self.status_file,
             config=self.event_generator_config,
+            seed=self.run_seed,
         )
         unused_globals = (
             set(self.contexts[0].global_names()) - event_generator.used_globals()
@@ -576,29 +589,35 @@ def build_event_generator(self, phasespaces: list[PhaseSpace]) -> ms.EventGenera
         return event_generator
 
     def survey_phasespaces(
-        self, phasespaces: list[PhaseSpace | None]
+        self, phasespaces: list[PhaseSpace | None], survey_pass: int = 0
     ) -> ms.EventGenerator | None:
         ps_filtered = [ps for ps in phasespaces if ps is not None]
         if len(ps_filtered) == 0:
             return None
         event_generator = self.build_event_generator(ps_filtered)
-        event_generator.survey()
+        event_generator.survey(survey_pass)
         return event_generator
 
     def survey(self) -> None:
+        # survey_pass distinguishes the survey() calls below: "both" mode can
+        # re-survey a channel carried over unchanged from the multichannel pass
+        # into the final (simplified) pass, and both passes schedule jobs on the
+        # same underlying ChannelEventGenerator. The explicit pass index keeps each
+        # pass's job seeds independent of the other passes' job counts, rather than
+        # depending on call history.
         phasespace_mode = self.run_card["phasespace"]["mode"]
         if phasespace_mode in ["multichannel", "both", "auto"]:
             self.phasespaces = [
                 subproc.build_multichannel_phasespace()
                 for subproc in self.subprocesses
             ]
-            self.event_generator = self.survey_phasespaces(self.phasespaces)
+            self.event_generator = self.survey_phasespaces(self.phasespaces, 0)
         elif phasespace_mode == "flat":
             self.phasespaces = [
                 subproc.build_flat_phasespace()
                 for subproc in self.subprocesses
             ]
-            self.event_generator = self.survey_phasespaces(self.phasespaces)
+            self.event_generator = self.survey_phasespaces(self.phasespaces, 0)
         else:
             raise ValueError("Unknown phasespace mode")
 
@@ -610,8 +629,8 @@ def survey(self) -> None:
             variance = 0.
             count_opt = 0
             for status in channel_status[chan_offset:chan_offset + len(ps.channels)]:
-                mean += status.mean
-                variance += status.error**2
+                mean += status.mean_abs
+                variance += status.error_abs**2
                 count_opt += status.count_opt
             rsd = (variance * count_opt)**0.5 / mean
             subproc.set_madnis_auto_settings(rsd)
@@ -644,7 +663,10 @@ def survey(self) -> None:
             ps_multi is not ps_both
             for ps_multi, ps_both in zip(phasespaces_multi, self.phasespaces)
         ):
-            self.event_generator = self.survey_phasespaces(self.phasespaces)
+            # distinct survey_pass: a channel carried over unchanged from the
+            # multichannel pass (pass 0) into this resurvey must not share its
+            # seed stream with that earlier pass.
+            self.event_generator = self.survey_phasespaces(self.phasespaces, 1)
 
     def train_madnis(self) -> None:
         madnis_args = self.run_card["madnis"]
@@ -715,6 +737,11 @@ def train_madnis(self) -> None:
             training_args=training_args,
             verbosity=verbosity,
             status_file=self.status_file,
+            # Reuses the run's resolved seed (also used by build_event_generator()).
+            # Only the single-channel CPU sample-generation path is currently seeded
+            # -- buffered training and GPU multi-channel batches are still
+            # non-deterministic.
+            seed=self.run_seed,
         )
         madnis_training.train()
         for phasespace, active_channels in zip(
@@ -918,6 +945,9 @@ def build_lhe_meta(self):
                 headers.append(ms.LHEHeader(name="MG5ProcCard", content=f.read()))
         headers.append(ms.LHEHeader(name="slha", content=param_text))
         headers.append(ms.LHEHeader(name="MG7RunCard", content=run_text))
+        # The resolved seed (even when the run_card requested a random one via
+        # seed = -1), so the run can be reproduced from the LHE file alone.
+        headers.append(ms.LHEHeader(name="MG7Seed", content=str(self.run_seed)))
         return ms.LHEMeta(
             beam1_pdg_id=beam_pdgs[0], beam2_pdg_id=beam_pdgs[1],
             beam1_energy=energies[0], beam2_energy=energies[1],
@@ -1678,6 +1708,9 @@ def set_madnis_auto_settings(self, rsd: float):
 
     def build_madnis(self, phasespace: PhaseSpace) -> PhaseSpace:
         madnis_args = self.process.run_card["madnis"]
+        # Shared across all networks below: initialize_globals() derives an
+        # independent, non-colliding stream per tensor from this one base seed.
+        seed = self.process.run_seed
         channels = []
         for channel_id, channel in enumerate(phasespace.channels):
             prefix = f"subproc{self.subproc_id}.channel{channel_id}"
@@ -1695,10 +1728,11 @@ def build_madnis(self, phasespace: PhaseSpace) -> PhaseSpace:
                 invert_spline=madnis_args["flow_invert_spline"],
             )
             if channel.adaptive_mapping is None:
-                flow.initialize_globals(self.process.contexts[0])
+                flow.initialize_globals(self.process.contexts[0], seed)
             else:
                 flow.initialize_from_vegas(
-                    self.process.contexts[0], channel.adaptive_mapping.grid_name()
+                    self.process.contexts[0], channel.adaptive_mapping.grid_name(),
+                    seed
                 )
             cond_dim += flow_dim
 
@@ -1715,7 +1749,7 @@ def build_madnis(self, phasespace: PhaseSpace) -> PhaseSpace:
                     subnet_layers=madnis_args["discrete_layers"],
                     subnet_activation=self.activation(madnis_args["discrete_activation"]),
                 )
-                discrete_sym.initialize_globals(self.process.contexts[0])
+                discrete_sym.initialize_globals(self.process.contexts[0], seed)
                 cond_dim += perm_count
 
             discrete_flavor = channel.discrete_flavor
@@ -1729,7 +1763,7 @@ def build_madnis(self, phasespace: PhaseSpace) -> PhaseSpace:
                     subnet_layers=madnis_args["discrete_layers"],
                     subnet_activation=self.activation(madnis_args["discrete_activation"]),
                 )
-                discrete_flavor.initialize_globals(self.process.contexts[0])
+                discrete_flavor.initialize_globals(self.process.contexts[0], seed)
 
             channels.append(Channel(
                 phasespace_mapping = channel.phasespace_mapping,
@@ -1803,7 +1837,9 @@ def build_cwnet(self, channel_count: int) -> ms.ChannelWeightNetwork:
             activation=self.activation(madnis_args["cwnet_activation"]),
             prefix=f"subproc{self.subproc_id}.cwnet",
         )
-        cwnet.initialize_globals(self.process.contexts[0])
+        cwnet.initialize_globals(
+            self.process.contexts[0], self.process.run_seed
+        )
         return cwnet
 
     def t_channel_mode(self, name: str) -> ms.PhaseSpaceMapping.TChannelMode:

madgraph/iolibs/template_files/mg7/run_card.toml

@@ -1,5 +1,11 @@
 [run]
 run_name = %(run.run_name)s
+# every run is reproducible: the same seed reproduces the run bit-identically,
+# and distinct seeds are statistically independent. -1 draws a fresh random
+# seed each run instead of fixing one here; the seed actually used is still
+# recorded (the MG7Seed tag in the LHE file, or the info.json status file), so
+# the run can be reproduced later.
+seed = %(run.seed)s
 # options: cuda, hip, cpp, cppnone, cppsse4, cppavx2, cpp512y, cpp512z, cppauto
 devices = %(run.devices)s
 # options:

madgraph/various/banner.py

@@ -6463,6 +6463,12 @@ def default_setup(self):
 
         # ----------------------------- [run] --------------------------
         self.add_toml_param('run', 'run_name', "run", gridpack=True)
+        self.add_toml_param('run', 'seed', -1, gridpack=True,
+            comment="every run is reproducible: the same seed reproduces the run "
+                    "bit-identically. -1 draws a fresh random seed each run instead "
+                    "of fixing one here; the seed actually used is still recorded "
+                    "(the MG7Seed tag in the LHE file, or the info.json status "
+                    "file), so the run can be reproduced later")
         self.add_toml_param('run', 'devices', ["cppnone"], typelist=str, gridpack=True,
             comment="options: cuda, hip, cpp, cppnone, cppsse4, cppavx2, cpp512y, cpp512z, cppauto")
         self.add_toml_param('run', 'simd_vector_size', -1,

madspace/.clang-format-ignore

@@ -7,3 +7,5 @@ src/cpu/runtime_mixin.inc
 src/gpu/runtime_mixin.inc
 src/cpu/runtime_backward_mixin.inc
 src/gpu/runtime_backward_mixin.inc
+include/madspace/mixmax/mixmax.hpp
+include/madspace/mixmax/mixmax_skip_N17.c