Bluebird gymnasium test agents

bluebird-gymnasium/bluebird_gymnasium/__init__.py

@@ -18,3 +18,5 @@
 register(id="SectorYEnv-v0", entry_point="bluebird_gymnasium.envs:SectorYEnv")
 
 register(id="SpringfieldEnv-v0", entry_point="bluebird_gymnasium.envs:SpringfieldEnv")
+
+register(id="FlightSchoolEnv-v0", entry_point="bluebird_gymnasium.envs:FlightSchoolEnv")

bluebird-gymnasium/bluebird_gymnasium/envs/__init__.py

@@ -152,13 +152,15 @@ class EnvConfig:
 
 # aircraft scenario generator class(es)
 from bluebird_dt.scenario_manager import (  # noqa: E402
+    Infinite,
     Regular,
     Tactical,
     TwoAircraft,
 )
 from bluebird_dt.scenario_manager.scenario_manager import ScenarioManager  # noqa: E402
 
 SCENARIO_CLS: dict[str, ScenarioManager] = {
+    "infinite": Infinite,
     "regular": Regular,
     "tactical": Tactical,
     "twoaircraft": TwoAircraft,
@@ -167,6 +169,7 @@ class EnvConfig:
 
 # now envs module imports
 from bluebird_gymnasium.envs.base import BaseEnv  # noqa: E402
+from bluebird_gymnasium.envs.flight_school import FlightSchoolEnv  # noqa: E402
 from bluebird_gymnasium.envs.infinite import CustomInfiniteEnv, InfiniteEnv  # noqa: E402
 from bluebird_gymnasium.envs.sector_i import SectorIEnv  # noqa: E402
 from bluebird_gymnasium.envs.sector_x import SectorXEnv  # noqa: E402
@@ -183,6 +186,7 @@ class EnvConfig:
     "SectorXPlusEnv-v0": SectorXPlusEnv,
     "SectorYEnv-v0": SectorYEnv,
     "SpringfieldEnv-v0": SpringfieldEnv,
+    "FlightSchoolEnv-v0": FlightSchoolEnv,
 }
 
 name_to_gym_key: dict[str, str] = {
@@ -193,6 +197,7 @@ class EnvConfig:
     "SectorXPlusEnv-v0": "sector_xplus",
     "SectorYEnv-v0": "sector_y",
     "SpringfieldEnv-v0": "springfield",
+    "FlightSchoolEnv-v0": "flight_school",
 }
 
 available_names = ", ".join(name_to_gym_key.values())
@@ -238,6 +243,7 @@ def get_env_cls_and_config(env_name: str) -> tuple[type[BaseEnv], EnvConfig]:
 __all__ = [
     "BaseEnv",
     "CustomInfiniteEnv",
+    "FlightSchoolEnv",
     "InfiniteEnv",
     "SectorIEnv",
     "SectorXEnv",

bluebird-gymnasium/bluebird_gymnasium/envs/flight_school.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import datetime
+
+from bluebird_dt.simulator import Simulator
+
+from bluebird_gymnasium.envs import EnvConfig, ViewType
+from bluebird_gymnasium.envs.sector_xplus import SectorXPlusEnv
+
+
+class FlightSchoolEnv(SectorXPlusEnv):
+    """Gymnasium environment for the Flight School scenario."""
+
+    def _generate_scenario(self) -> Simulator:
+        category = "Flight School"
+        scenario = "Xplus-Sector"
+        timestamp = datetime.datetime.now().strftime("%Y_%m_%d__%H_%M_%S")
+
+        suffix = self.config.simulation_log_config.get("log_suffix", None)
+        suffix = "" if suffix is None or suffix == "" else f"__{suffix}"
+        log_filename = f"{category}_{scenario}_{timestamp}{suffix}"
+
+        return self.scenario_manager.to_simulator(
+            category=category,
+            scenario_name=scenario,
+            save_log_to_file=False,
+            log_filename=log_filename,
+            predictor=None,
+        )
+
+    @classmethod
+    def get_default_env_config(cls, view_type: ViewType | str = ViewType.CENTRALIZED) -> EnvConfig:
+        config = super().get_default_env_config(view_type)
+        config.scenario_config = {
+            "cls": "infinite",
+            "args": {
+                "random_seed": None,
+                "num_starter_aircraft": 2,
+                "initial_spawn_rate": 0.002,
+                "spawn_rate_increment": 0.002,
+                "spawn_rate_increase_interval": 60,
+                "max_spawn_rate": 0.03,
+                "total_time_seconds": 3600.0,
+            },
+        }
+        config.reward_config = {
+            "fns": [
+                "position_status_const",
+                "lateral_centreline_distance_shaped",
+                "safety_simple_avoidance_exp",
+            ],
+            "coeffs": [1.0, 1.0, 1.2],
+        }
+        config.scenario_duration = 10 * 60
+        return config

bluebird-gymnasium/bluebird_gymnasium/rewards/__init__.py

@@ -98,3 +98,20 @@
 mod_name = f"{base_pkg}.custom.reward_drlan"
 registry_reward_fn.register("reward_drlan", f"{mod_name}:reward_drlan")
 registry_reward_fn.register("custom_reward", f"{mod_name}:custom_reward_fn")
+mod_name = f"{base_pkg}.custom.custom_reward"
+registry_reward_fn.register(
+    "route_progress_terminal_reward",
+    f"{mod_name}:route_progress_terminal_reward",
+)
+registry_reward_fn.register(
+    "lateral_termination_check_sac_env",
+    f"{mod_name}:lateral_termination_check_sac_env",
+)
+registry_reward_fn.register(
+    "lateral_termination_check_mac_env",
+    f"{mod_name}:lateral_termination_check_mac_env",
+)
+registry_reward_fn.register(
+    "anti_loiter_route_rejoin_reward",
+    f"{mod_name}:anti_loiter_route_rejoin_reward",
+)

bluebird-gymnasium/bluebird_gymnasium/rewards/custom/custom_reward.py

@@ -1,4 +1,11 @@
+from typing import Any
+
 from bluebird_gymnasium.envs.base import BaseEnv
+from bluebird_gymnasium.rewards.lateral_termination_check import (
+    lateral_termination_check_mac,
+    lateral_termination_check_sac,
+)
+from bluebird_gymnasium.utils.types import PositionStatus
 
 
 def custom_reward_fn(gym_env: BaseEnv, callsign: str, action: int, **kwargs) -> float:  # noqa: ARG001, ANN003
@@ -16,3 +23,151 @@ def custom_reward_fn(gym_env: BaseEnv, callsign: str, action: int, **kwargs) ->
 
     # implement reward function here.
     return 0.0
+
+
+def route_progress_terminal_reward(gym_env: BaseEnv, callsign: str) -> float:
+    """Reward progress toward exit and penalize stalling at episode end.
+
+    This reward is intended for the PPO curriculum notebooks where the agent
+    was observed to loiter near its entry fix. It combines three signals:
+
+    - a positive terminal bonus for correctly reaching the exit window
+    - a per-step progress reward for reducing route-track distance to exit
+    - a timeout penalty if the episode ends with the aircraft still far away
+
+    Returns:
+        float: shaped reward that encourages forward progress through sector.
+    """
+
+    reward = 0.0
+
+    ac_tracked_state = gym_env.get_tracked_aircraft_data(callsign)
+    prev_ac_tracked_state = gym_env.get_tracked_aircraft_data_previous(callsign)
+
+    if ac_tracked_state is None:
+        return reward
+
+    if ac_tracked_state.pos_status == PositionStatus.EXIT_REACHED:
+        reward += 20.0
+
+    if (
+        prev_ac_tracked_state is not None
+        and ac_tracked_state.track_dist_to_exit_cr is not None
+        and prev_ac_tracked_state.track_dist_to_exit_cr is not None
+    ):
+        curr_dist = float(ac_tracked_state.track_dist_to_exit_cr)
+        prev_dist = float(prev_ac_tracked_state.track_dist_to_exit_cr)
+        distance_improvement = prev_dist - curr_dist
+
+        # Positive when the aircraft gets closer to its exit, negative when it
+        # drifts away or loiters. Clip to keep this component bounded.
+        reward += max(min(distance_improvement / 5.0, 2.0), -2.0)
+
+    if (
+        gym_env.timestep >= gym_env.maxstep
+        and ac_tracked_state.track_dist_to_exit_cr is not None
+        and ac_tracked_state.pos_status != PositionStatus.EXIT_REACHED
+    ):
+        timeout_distance = float(ac_tracked_state.track_dist_to_exit_cr)
+        reward -= min(timeout_distance / 20.0, 10.0)
+
+    return float(reward)
+
+
+def lateral_termination_check_sac_env(
+    gym_env: BaseEnv, callsign: str, action: int, **kwargs: dict[str, dict[str, Any]]
+) -> float:
+    """Notebook-safe wrapper for the SAC lateral termination reward.
+
+    The base reward registry invokes reward functions with only
+    `(gym_env, callsign, action)`. This wrapper adapts the original helper,
+    which expects explicit `timestep` and `maxstep` arguments.
+    """
+
+    return float(
+        lateral_termination_check_sac(
+            gym_env=gym_env,
+            callsign=callsign,
+            action=action,
+            timestep=gym_env.timestep,
+            maxstep=gym_env.maxstep,
+            **kwargs,
+        )
+    )
+
+
+def lateral_termination_check_mac_env(
+    gym_env: BaseEnv, callsign: str, action: int, **kwargs: dict[str, dict[str, Any]]
+) -> float:
+    """Notebook-safe wrapper for the MAC lateral termination reward."""
+
+    ac_tracked_state = gym_env.get_tracked_aircraft_data(callsign)
+    transferred = ac_tracked_state is not None and ac_tracked_state.pos_status == PositionStatus.EXIT_REACHED
+
+    return float(
+        lateral_termination_check_mac(
+            gym_env=gym_env,
+            callsign=callsign,
+            action=action,
+            timestep=gym_env.timestep,
+            maxstep=gym_env.maxstep,
+            transferred=transferred,
+            **kwargs,
+        )
+    )
+
+
+def anti_loiter_route_rejoin_reward(gym_env: BaseEnv, callsign: str, action: int) -> float:
+    """Reward progress and route rejoin, penalize stalling near the entry.
+
+    This is used by the PPO curriculum notebook to address a specific failure
+    mode seen in multi-aircraft stages: orbiting near the start fix to avoid
+    future conflict instead of making progress, taking an avoidance action,
+    then returning to the route.
+    """
+
+    simulator_env = gym_env.get_simulator_env()
+    aircraft = simulator_env.aircraft[callsign]
+
+    ac_tracked_state = gym_env.get_tracked_aircraft_data(callsign)
+    prev_ac_tracked_state = gym_env.get_tracked_aircraft_data_previous(callsign)
+
+    if ac_tracked_state is None or prev_ac_tracked_state is None:
+        return 0.0
+
+    curr_exit_dist = ac_tracked_state.track_dist_to_exit_cr
+    prev_exit_dist = prev_ac_tracked_state.track_dist_to_exit_cr
+
+    if curr_exit_dist is None or prev_exit_dist is None:
+        return 0.0
+
+    reward = 0.0
+
+    progress_nm = float(prev_exit_dist - curr_exit_dist)
+
+    curr_centre_dist = None
+    prev_centre_dist = None
+    if ac_tracked_state.centreline_info_fr is not None:
+        curr_centre_dist = float(ac_tracked_state.centreline_info_fr[0])
+    if prev_ac_tracked_state.centreline_info_fr is not None:
+        prev_centre_dist = float(prev_ac_tracked_state.centreline_info_fr[0])
+
+    if progress_nm > 0.15:
+        reward += min(progress_nm / 2.0, 1.0)
+    elif float(curr_exit_dist) > 20.0:
+        reward -= 0.35
+
+    if curr_centre_dist is not None and prev_centre_dist is not None:
+        centreline_improvement = prev_centre_dist - curr_centre_dist
+        if centreline_improvement > 0.1:
+            reward += min(centreline_improvement / 2.0, 0.75)
+        elif curr_centre_dist > 8.0 and action == 0:
+            reward -= 0.25
+
+        if curr_centre_dist < 3.0 and progress_nm > 0.15:
+            reward += 0.4
+
+    if aircraft.on_route and progress_nm > 0.15:
+        reward += 0.25
+
+    return float(reward)

bluebird-gymnasium/bluebird_gymnasium/rewards/expeditious.py

@@ -75,9 +75,9 @@ def expeditious_linear(gym_env: BaseEnv, callsign: str, action: int, **kwargs) -
     else:
         curr_dist = ac_tracked_state.track_dist_to_exit_cr
         prev_dist = prev_ac_tracked_state.track_dist_to_exit_cr
-        dist_diff = curr_dist - prev_dist
+        distance_improvement = prev_dist - curr_dist
 
-        reward = np.clip(dist_diff, -DIFF_THRESHOLD, DIFF_THRESHOLD)
+        reward = np.clip(distance_improvement, -DIFF_THRESHOLD, DIFF_THRESHOLD)
 
         # scale the reward based on the aircraft's speed. (i.e., it is
         # more impressive for an aircraft with a slower speed to travel
@@ -136,9 +136,9 @@ def expeditious_quad(gym_env: BaseEnv, callsign: str, action: int, **kwargs) ->
     else:
         curr_dist = ac_tracked_state.track_dist_to_exit_cr
         prev_dist = prev_ac_tracked_state.track_dist_to_exit_cr
-        dist_diff = curr_dist - prev_dist
+        distance_improvement = prev_dist - curr_dist
 
-        reward = np.clip(dist_diff, -DIFF_THRESHOLD, DIFF_THRESHOLD)
+        reward = np.clip(distance_improvement, -DIFF_THRESHOLD, DIFF_THRESHOLD)
         sign = np.sign(reward)
         reward = 1.5 * (reward**2)
 

bluebird-gymnasium/bluebird_gymnasium/rewards/lateral_next_fix_proximity.py

@@ -123,4 +123,10 @@ def lateral_next_fix_proximity_dist_exp(gym_env: BaseEnv, callsign: str, action:
     pf_nf_dist = prev_fix.distance(next_fix)
     ac_nf_dist = aircraft.distance(next_fix)
 
+    # Some scenarios can yield a degenerate route segment where the previous
+    # and next fixes coincide. Fall back to a direct exponential in that case
+    # instead of dividing by zero.
+    if pf_nf_dist <= 1e-6:
+        return np.exp(-ac_nf_dist)
+
     return np.exp(-ac_nf_dist / pf_nf_dist)