diff --git a/source/isaaclab_tasks/changelog.d/jichuanh-stabilize-rendering-mdl-warmup.rst b/source/isaaclab_tasks/changelog.d/jichuanh-stabilize-rendering-mdl-warmup.rst new file mode 100644 index 000000000000..904156117865 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/jichuanh-stabilize-rendering-mdl-warmup.rst @@ -0,0 +1,23 @@ +Fixed +^^^^^ + +* Stabilized RTX MDL shader-warmup flakes in the rendering-correctness tests + by polling the camera output until every data-type tensor reports a + non-zero max before the assertion / golden compare: + + * ``test_camera_renders_not_empty`` in + ``test_shadow_hand_vision_presets.py`` polls via ``env.step()`` with a + 60-step cap. + * ``rendering_test_utils.warmup_render_until_nonzero`` is invoked from the + four rendering helpers in ``rendering_test_utils.py`` + (``rendering_test_shadow_hand`` / ``_cartpole`` / ``_dexsuite_kuka``) and + from ``test_rendering_registered_tasks.py``. It iterates over every + sensor in ``env.scene.sensors`` and polls via ``sim.render()`` + + ``scene.update()`` with a 30-pass cap. Physics state is not advanced, so + the existing golden images stay valid. + + All affected variants intermittently failed with "Camera output is all + zeros or all inf" for ``simple_shading_*_mdl`` and + ``simple_shading_constant_diffuse`` on cold-cache CI runners because the + GPU returned a still-zero framebuffer before the MDL material finished + compiling. diff --git a/source/isaaclab_tasks/test/rendering_test_utils.py b/source/isaaclab_tasks/test/rendering_test_utils.py index d4fbf368ea7c..499b72e3972a 100644 --- a/source/isaaclab_tasks/test/rendering_test_utils.py +++ b/source/isaaclab_tasks/test/rendering_test_utils.py @@ -675,6 +675,51 @@ def validate_camera_outputs( pytest.fail(reason) +def warmup_render_until_nonzero(env, max_passes: int = 30) -> None: + """Drive extra render passes until every camera output tensor has a non-zero max. + + RTX MDL materials compile asynchronously on first use. The single render pass that env + construction triggers (via ``scene.update`` in ``__init__``) is not always enough — on + cold-cache CI runners the GPU returns a still-zero framebuffer for shader variants that + have not finished compiling, which trips :func:`validate_camera_outputs` with + "no non-zero pixels" or "all zeros or all inf". + + Polling exits as soon as every camera output is ready, so the scene state stays at the + same "first non-zero frame" the existing goldens were captured at. Renders are driven by + ``sim.render()`` + ``scene.update()`` (no ``env.step()``) so physics state is not + advanced. This mirrors the pattern used by + :attr:`~isaaclab.envs.DirectRLEnvCfg.num_rerenders_on_reset` and + :attr:`~isaaclab.envs.DirectRLEnvCfg.wait_for_textures` in the core env classes. + """ + base = getattr(env, "unwrapped", env) + scene = getattr(base, "scene", None) + if scene is None or not getattr(scene, "sensors", None): + return + + physics_dt = base.physics_dt + for _ in range(max_passes): + outputs_ready = True + for sensor in scene.sensors.values(): + data = getattr(sensor, "data", None) + output = getattr(data, "output", None) if data is not None else None + if not isinstance(output, dict): + continue + for value in output.values(): + tensor = value if isinstance(value, torch.Tensor) else getattr(value, "torch", None) + if tensor is None: + continue + finite = torch.where(torch.isinf(tensor), torch.zeros_like(tensor), tensor) + if finite.max() <= 0.2: + outputs_ready = False + break + if not outputs_ready: + break + if outputs_ready: + return + base.sim.render() + scene.update(dt=physics_dt) + + def rendering_test_shadow_hand( physics_backend: str, renderer: str, @@ -700,6 +745,7 @@ def rendering_test_shadow_hand( try: env = ShadowHandVisionEnv(env_cfg) maybe_save_stage("shadow_hand", physics_backend, renderer, data_type) + warmup_render_until_nonzero(env) validate_camera_outputs( "shadow_hand", @@ -739,6 +785,7 @@ def rendering_test_cartpole( try: env = CartpoleCameraEnv(env_cfg) maybe_save_stage("cartpole", physics_backend, renderer, data_type) + warmup_render_until_nonzero(env) validate_camera_outputs( "cartpole", physics_backend, @@ -795,6 +842,7 @@ def rendering_test_dexsuite_kuka( try: env = ManagerBasedRLEnv(env_cfg) maybe_save_stage("dexsuite_kuka", physics_backend, renderer, data_type) + warmup_render_until_nonzero(env) validate_camera_outputs( "dexsuite_kuka", physics_backend, diff --git a/source/isaaclab_tasks/test/test_rendering_registered_tasks.py b/source/isaaclab_tasks/test/test_rendering_registered_tasks.py index 94eb6e07232a..a5b7da879ed0 100644 --- a/source/isaaclab_tasks/test/test_rendering_registered_tasks.py +++ b/source/isaaclab_tasks/test/test_rendering_registered_tasks.py @@ -24,6 +24,7 @@ make_generate_html_report_fixture, maybe_save_stage, validate_camera_outputs, + warmup_render_until_nonzero, ) pytestmark = pytest.mark.isaacsim_ci @@ -99,6 +100,7 @@ def test_rendering_registered_tasks(task_id: str, env_name: str): sim._app_control_on_stop_handle = None maybe_save_stage(f"registered_tasks_{task_id}", "default_physics", "default_renderer", "stage") + warmup_render_until_nonzero(env) camera_outputs_nested_dict = _collect_camera_outputs(env) num_camera_outputs = len(camera_outputs_nested_dict) diff --git a/source/isaaclab_tasks/test/test_shadow_hand_vision_presets.py b/source/isaaclab_tasks/test/test_shadow_hand_vision_presets.py index ccec60c3fa46..2b5ce3851991 100644 --- a/source/isaaclab_tasks/test/test_shadow_hand_vision_presets.py +++ b/source/isaaclab_tasks/test/test_shadow_hand_vision_presets.py @@ -398,7 +398,21 @@ def render_correctness_env(request, shadow_hand_vision_presets): env = ShadowHandVisionEnv(cfg) env.reset() actions = torch.zeros(cfg.scene.num_envs, env.action_space.shape[-1], device=env.device) - env.step(actions) + # Step until all camera outputs are non-zero (RTX MDL shaders compile lazily). + # ``simple_shading_*_mdl`` presets can take 10–30 frames to produce non-zero pixels + # on cold-cache CI runners; poll up to ``_MAX_WARMUP_STEPS`` and exit early once ready. + _MAX_WARMUP_STEPS = 60 + for _ in range(_MAX_WARMUP_STEPS): + env.step(actions) + outputs_ready = True + for output in env._tiled_camera.data.output.values(): + tensor = output.torch + finite = torch.where(torch.isinf(tensor), torch.zeros_like(tensor), tensor) + if finite.max() <= 0.2: + outputs_ready = False + break + if outputs_ready: + break yield renderer_preset, camera_preset, physics, env env.close()