Add lumen-gi-enabled / lumen-reflections-enabled camera capture settings (captures match the viewport by default)

client/python/projectairsim/src/projectairsim/schema/robot_config_schema.jsonc

@@ -664,7 +664,9 @@
           "streaming-enabled": { "type": "boolean" },
           "pixels-as-float": { "type": "boolean" },
           "compress": { "type": "boolean" },
-          "target-gamma": { "type": "number" }
+          "target-gamma": { "type": "number" },
+          "lumen-gi-enabled": { "type": "boolean" },
+          "lumen-reflections-enabled": { "type": "boolean" }
         }
       }
     },

core_sim/include/core_sim/sensors/camera.hpp

@@ -101,6 +101,23 @@ struct CaptureSettings {
   float auto_exposure_histogram_log_max =
       std::numeric_limits<float>::quiet_NaN();  // 4;
   float motion_blur_amount = std::numeric_limits<float>::quiet_NaN();
+  // Lumen dynamic GI / reflections for this capture's scene render (UE 5.5+
+  // supports Lumen in scene captures). Tri-state, kLumenAuto when the key is
+  // absent from the config:
+  //   auto: Scene (RGB) captures mirror the project's methods
+  //         (r.DynamicGlobalIlluminationMethod / r.ReflectionMethod) so the
+  //         captured image matches the viewport; every other image type
+  //         resolves to None (their output is a replacement material — GI
+  //         underneath is pure GPU/VRAM waste).
+  //   1:    force Lumen for this capture.
+  //   0:    force the method to None. This must be an active override — an
+  //         un-overridden capture inherits the project CVar, so "no override"
+  //         does not mean "no Lumen".
+  // Each Lumen-enabled capture maintains its own Lumen scene (GPU time +
+  // VRAM cost).
+  static constexpr int kLumenAuto = -1;
+  int lumen_gi_enabled = kLumenAuto;
+  int lumen_reflections_enabled = kLumenAuto;
   float target_gamma =
       2.5f;  // This would be reset to kSceneTargetGamma for scene as default
   int projection_mode = 0;  // ECameraProjectionMode::Perspective

core_sim/src/constant.hpp

@@ -202,6 +202,9 @@ class Constant {
     static constexpr const char* auto_exposure_histogram_log_max =
         "auto-exposure-histogram-log-max";
     static constexpr const char* motion_blur_amount = "motion-blur-amount";
+    static constexpr const char* lumen_gi_enabled = "lumen-gi-enabled";
+    static constexpr const char* lumen_reflections_enabled =
+        "lumen-reflections-enabled";
     static constexpr const char* target_gamma = "target-gamma";
     static constexpr const char* max_depth_meters = "max-depth-meters";
     static constexpr const char* chromatic_aberration_intensity = "chromatic-aberration-intensity";

core_sim/src/sensors/camera.cpp

@@ -1345,6 +1345,22 @@ void Camera::Loader::LoadCaptureSetting(const json& json) {
   setting.motion_blur_amount = JsonUtils::GetNumber<float>(
       json, Constant::Config::motion_blur_amount, setting.motion_blur_amount);
 
+  // Tri-state: absent key = kLumenAuto (match the project/viewport method),
+  // explicit true/false forces the method on/off for this capture.
+  if (JsonUtils::HasKey(json, Constant::Config::lumen_gi_enabled)) {
+    setting.lumen_gi_enabled =
+        JsonUtils::GetBoolean(json, Constant::Config::lumen_gi_enabled, false)
+            ? 1
+            : 0;
+  }
+  if (JsonUtils::HasKey(json, Constant::Config::lumen_reflections_enabled)) {
+    setting.lumen_reflections_enabled =
+        JsonUtils::GetBoolean(json, Constant::Config::lumen_reflections_enabled,
+                              false)
+            ? 1
+            : 0;
+  }
+
   setting.target_gamma = JsonUtils::GetNumber<float>(
       json, Constant::Config::target_gamma, setting.target_gamma);
 

docs/sensors/camera_capture_settings.md

@@ -96,6 +96,8 @@ A single camera can output multiple capture types. The following parameters desc
 | auto-exposure-histogram-log-min |                                   | Defines the lower bounds for the brightness range of the generated histogram when using the  HDR (Eye Adaptation) visualization mode;log-min value for histogram of scene colors; E.g.:0.8 ==> 80% of the screen pixels are darker than the luminance value A |
 | auto-exposure-histogram-log-max |                                   | log-max value for histogram of scene colors; E.g.:0.95 ==>  95% of the screen pixels are darker than the luminance value B. Current luminance value (C) = avg(A, B) |
 | motion-blur-amount              | [0, 1]                            | Blurs objects based on it's motion determined using velocity maps. A value of 0.25 - 0.5 looks reasonable. Avoid 1.0 |
+| lumen-gi-enabled                | true/false (default: match project setting) | Controls Lumen dynamic global illumination for this capture. **When omitted**, RGB Scene captures mirror the project's default global illumination method (the `r.DynamicGlobalIlluminationMethod` project setting) — i.e. Lumen projects get Lumen captures by default — while non-Scene image types (depth, segmentation, ...) run with no dynamic GI, since their output is a replacement material and GI underneath only costs GPU/VRAM. Set `false` to force a capture back to no dynamic GI, or `true` to force Lumen regardless of the project default. **Note:** capture views never blend Post Process Volumes, so if your level selects or disables a GI method through a PPV (bounded or unbound) rather than the project setting, the default will not mirror that — set this explicitly. Requires UE 5.5+ (compiled out on older engines, which hard-disable Lumen for captures; legacy behavior is kept there); each Lumen-enabled capture maintains its own Lumen scene (GPU time + VRAM cost). |
+| lumen-reflections-enabled       | true/false (default: match project setting) | Same tri-state semantics and PPV caveat as `lumen-gi-enabled`, but for the reflection method (`r.ReflectionMethod`). |
 | target-gamma                    |                                   | Target gamma to be used by the render target (UTextureRenderTarget2D) |
 | max-depth-meters              | > 0.0                             | Maximum depth for Depth Visualization. Objects beyond the maximum depth will be true white, while close objects will be completely black. |
 

unreal/Blocks/Plugins/ProjectAirSim/Source/ProjectAirSim/Private/Sensors/UnrealCamera.cpp

@@ -18,6 +18,7 @@
 #include "Engine/TextureRenderTarget2D.h"
 #include "Engine/World.h"
 #include "GameFramework/PlayerController.h"
+#include "HAL/IConsoleManager.h"
 #include "ImagePackingAsyncTask.h"
 #include "ImageUtils.h"
 #include "Materials/MaterialInstanceDynamic.h"
@@ -168,6 +169,63 @@ void HideDebugDrawComponent(USceneCaptureComponent2D* CaptureComponent,
   #endif
 }
 
+#if UE_SUPPORTS_LUMEN_SCENE_CAPTURES
+// Resolves the tri-state lumen-gi-enabled / lumen-reflections-enabled
+// capture settings into the concrete methods to pin on this capture's
+// post-process overrides:
+//   auto (key absent): Scene (RGB) captures mirror the project's DEFAULT
+//     methods (r.DynamicGlobalIlluminationMethod / r.ReflectionMethod);
+//     every other image type resolves to None — their output comes from a
+//     replacement material, so dynamic GI underneath is pure GPU/VRAM waste
+//     (each Lumen-enabled capture maintains its own Lumen scene). Note the
+//     project default is not always what the viewport shows: capture views
+//     never blend Post Process Volumes, so a level that selects or disables
+//     a method through a PPV (bounded or unbound) is not mirrored — set the
+//     settings explicitly there.
+//   true:  force Lumen for this capture regardless of the project default.
+//   false: force None. This must be an active override — a capture's view
+//     inherits the project CVar when un-overridden, so doing nothing does
+//     not mean "no Lumen".
+static void ResolveCaptureLumenMethods(
+    const projectairsim::CaptureSettings& CaptureSettings,
+    EDynamicGlobalIlluminationMethod::Type& OutGIMethod,
+    EReflectionMethod::Type& OutReflectionMethod) {
+  const bool bIsSceneCapture =
+      CaptureSettings.image_type ==
+      projectairsim::MathUtils::ToNumeric(projectairsim::ImageType::kScene);
+
+  if (CaptureSettings.lumen_gi_enabled ==
+      projectairsim::CaptureSettings::kLumenAuto) {
+    static const auto* GIMethodCVar =
+        IConsoleManager::Get().FindTConsoleVariableDataInt(
+            TEXT("r.DynamicGlobalIlluminationMethod"));
+    OutGIMethod = (bIsSceneCapture && GIMethodCVar)
+                      ? static_cast<EDynamicGlobalIlluminationMethod::Type>(
+                            GIMethodCVar->GetValueOnGameThread())
+                      : EDynamicGlobalIlluminationMethod::None;
+  } else {
+    OutGIMethod = CaptureSettings.lumen_gi_enabled
+                      ? EDynamicGlobalIlluminationMethod::Lumen
+                      : EDynamicGlobalIlluminationMethod::None;
+  }
+
+  if (CaptureSettings.lumen_reflections_enabled ==
+      projectairsim::CaptureSettings::kLumenAuto) {
+    static const auto* ReflectionMethodCVar =
+        IConsoleManager::Get().FindTConsoleVariableDataInt(
+            TEXT("r.ReflectionMethod"));
+    OutReflectionMethod = (bIsSceneCapture && ReflectionMethodCVar)
+                              ? static_cast<EReflectionMethod::Type>(
+                                    ReflectionMethodCVar->GetValueOnGameThread())
+                              : EReflectionMethod::None;
+  } else {
+    OutReflectionMethod = CaptureSettings.lumen_reflections_enabled
+                              ? EReflectionMethod::Lumen
+                              : EReflectionMethod::None;
+  }
+}
+#endif  // UE_SUPPORTS_LUMEN_SCENE_CAPTURES
+
 void UUnrealCamera::CreateComponents() {
   Captures.Init(nullptr, ImageTypeCount());
   RenderTargets.Init(nullptr, ImageTypeCount());
@@ -431,6 +489,21 @@ void UUnrealCamera::UpdateCaptureComponentSetting(
   if (!std::isnan(CaptureSettings.target_gamma))
     RenderTarget->TargetGamma = CaptureSettings.target_gamma;
 
+#if UE_SUPPORTS_LUMEN_SCENE_CAPTURES
+  // Lumen needs ray tracing data in the capture's scene render when the
+  // project uses hardware ray tracing; without this flag the capture's rays
+  // miss everything even though the resolved GI method says Lumen. Harmless
+  // otherwise (software Lumen works without it).
+  {
+    EDynamicGlobalIlluminationMethod::Type GIMethod;
+    EReflectionMethod::Type ReflectionMethod;
+    ResolveCaptureLumenMethods(CaptureSettings, GIMethod, ReflectionMethod);
+    Capture->bUseRayTracingIfEnabled =
+        GIMethod == EDynamicGlobalIlluminationMethod::Lumen ||
+        ReflectionMethod == EReflectionMethod::Lumen;
+  }
+#endif
+
   Capture->ProjectionType =
       static_cast<ECameraProjectionMode::Type>(CaptureSettings.projection_mode);
 
@@ -449,6 +522,25 @@ void UUnrealCamera::UpdateCaptureComponentSetting(
 void UUnrealCamera::UpdateCameraPostProcessingSetting(
     FPostProcessSettings& PostProcessSettings,
     const projectairsim::CaptureSettings& CaptureSettings) {
+#if UE_SUPPORTS_LUMEN_SCENE_CAPTURES
+  // Pin this capture's global illumination / reflection method explicitly
+  // (see ResolveCaptureLumenMethods). When the method resolves to Lumen the
+  // engine maintains a dedicated Lumen scene for the capture's persistent
+  // view state — this is what makes the captured image match the Lumen-lit
+  // editor/game viewport instead of losing all indirect light. Compiled out
+  // below UE 5.5, where the engine hard-disables Lumen for captures and
+  // legacy behavior is kept exactly.
+  {
+    EDynamicGlobalIlluminationMethod::Type GIMethod;
+    EReflectionMethod::Type ReflectionMethod;
+    ResolveCaptureLumenMethods(CaptureSettings, GIMethod, ReflectionMethod);
+    PostProcessSettings.bOverride_DynamicGlobalIlluminationMethod = true;
+    PostProcessSettings.DynamicGlobalIlluminationMethod = GIMethod;
+    PostProcessSettings.bOverride_ReflectionMethod = true;
+    PostProcessSettings.ReflectionMethod = ReflectionMethod;
+  }
+#endif
+
   if (CaptureSettings.auto_exposure_method >= 0) {
     PostProcessSettings.bOverride_AutoExposureMethod = true;
     PostProcessSettings.AutoExposureMethod =

unreal/Blocks/Plugins/ProjectAirSim/Source/ProjectAirSim/Private/UnrealCompatibility.h

@@ -9,3 +9,10 @@
 
 // Convenience macro
 #define UE_IS_SUPPORTED (UE_IS_5_2 || UE_IS_5_7)
+
+// The engine supports Lumen in scene captures from UE 5.5 (it maintains a
+// dedicated Lumen scene per opted-in capture). Older engines hard-disable
+// Lumen for capture views, so the lumen-* capture settings are compiled out
+// below 5.5 and captures keep the exact legacy behavior there.
+#define UE_SUPPORTS_LUMEN_SCENE_CAPTURES \
+  (ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION >= 5)