From bfb179cda48784cabf068d300708552e5d5bc52e Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Tue, 16 Dec 2025 20:06:25 -0400 Subject: [PATCH 01/19] PPO Example (with checkpoints, tensorboard, GPU, custom reward fn) --- .../catanatron/gym/envs/catanatron_env.py | 19 +- examples/ppo/.gitignore | 2 + examples/ppo/shaped_reward.py | 94 ++++++++++ examples/ppo/train.py | 171 ++++++++++++++++++ 4 files changed, 278 insertions(+), 8 deletions(-) create mode 100644 examples/ppo/.gitignore create mode 100644 examples/ppo/shaped_reward.py create mode 100644 examples/ppo/train.py diff --git a/catanatron/catanatron/gym/envs/catanatron_env.py b/catanatron/catanatron/gym/envs/catanatron_env.py index a1f43db9..623978b6 100644 --- a/catanatron/catanatron/gym/envs/catanatron_env.py +++ b/catanatron/catanatron/gym/envs/catanatron_env.py @@ -114,7 +114,7 @@ def from_action_space(action_int, playable_actions): HIGH = 19 * 5 -def simple_reward(game, p0_color): +def simple_reward(action, game, p0_color): winning_color = game.winning_color() if p0_color == winning_color: return 1 @@ -133,6 +133,7 @@ class CatanatronEnv(gym.Env): metadata = {"render_modes": []} def __init__(self, config=None): + self.dtype = np.float32 self.config = config or dict() self.invalid_action_reward = self.config.get("invalid_action_reward", -1) self.reward_function = self.config.get("reward_function", simple_reward) @@ -156,14 +157,14 @@ def __init__(self, config=None): if self.representation == "mixed": channels = get_channels(len(self.players)) board_tensor_space = spaces.Box( - low=0, high=1, shape=(channels, 21, 11), dtype=np.float64 + low=0, high=1, shape=(channels, 21, 11), dtype=self.dtype ) self.numeric_features = [ f for f in self.features if not is_graph_feature(f) ] # TODO: This could be tigher (e.g. _ROADS_AVAILABLE <= 15) numeric_space = spaces.Box( - low=0, high=HIGH, shape=(len(self.numeric_features),), dtype=np.float64 + low=0, high=HIGH, shape=(len(self.numeric_features),), dtype=self.dtype ) mixed = spaces.Dict( { @@ -175,7 +176,7 @@ def __init__(self, config=None): else: # TODO: This could be tigher (e.g. _ROADS_AVAILABLE <= 15) self.observation_space = spaces.Box( - low=0, high=HIGH, shape=(len(self.features),), dtype=np.float64 + low=0, high=HIGH, shape=(len(self.features),), dtype=self.dtype ) self.reset() @@ -216,7 +217,7 @@ def step(self, action): winning_color = self.game.winning_color() terminated = winning_color is not None truncated = self.game.state.num_turns >= TURNS_LIMIT - reward = self.reward_function(self.game, self.p0.color) + reward = self.reward_function(catan_action, self.game, self.p0.color) return observation, reward, terminated, truncated, info @@ -251,10 +252,12 @@ def _get_observation(self) -> Union[np.ndarray, MixedObservation]: board_tensor = create_board_tensor( self.game, self.p0.color, channels_first=True ) - numeric = np.array([float(sample[i]) for i in self.numeric_features]) + numeric = np.array( + [sample[i] for i in self.numeric_features], dtype=self.dtype + ) return {"board": board_tensor, "numeric": numeric} - return np.array([float(sample[i]) for i in self.features]) + return np.array([sample[i] for i in self.features], dtype=self.dtype) def _advance_until_p0_decision(self): while ( @@ -264,7 +267,7 @@ def _advance_until_p0_decision(self): self.game.play_tick() # will play bot -CatanatronEnv.__doc__ = f""" +CatanatronEnv.__doc__ = """ 1v1 environment against a random player Attributes: diff --git a/examples/ppo/.gitignore b/examples/ppo/.gitignore new file mode 100644 index 00000000..469e870b --- /dev/null +++ b/examples/ppo/.gitignore @@ -0,0 +1,2 @@ +tensorboard_logs +checkpoints \ No newline at end of file diff --git a/examples/ppo/shaped_reward.py b/examples/ppo/shaped_reward.py new file mode 100644 index 00000000..2aea336d --- /dev/null +++ b/examples/ppo/shaped_reward.py @@ -0,0 +1,94 @@ +""" +Shaped reward function for Catan that provides incremental rewards. + +Instead of only rewarding at the end (+1 win, -1 loss), this gives +partial credit for progress during the game. +""" + +from catanatron.state_functions import ( + get_actual_victory_points, + get_longest_road_color, + get_largest_army, +) + + +class ShapedRewardFunction: + """ + Reward function that gives incremental rewards for game progress. + + Rewards: + - Victory point gain: +1.0 per VP + - Winning: +10.0 bonus + - Losing: -10.0 penalty + - Longest road acquired: +0.5 + - Largest army acquired: +0.5 + """ + + def __init__(self): + self.reset() + + def reset(self): + """Reset tracked state for a new game.""" + self.prev_vp = 0 + self.prev_has_longest_road = False + self.prev_has_largest_army = False + + def __call__(self, action, game, p0_color): + """ + Compute reward for the current step. + + Args: + action: The action taken + game: The game object + p0_color: Player 0's color (BLUE) + + Returns: + float: The reward for this step + """ + state = game.state + reward = 0.0 + + # Get current metrics + current_vp = get_actual_victory_points(state, p0_color) + longest_road_color = get_longest_road_color(state) + largest_army_color, _ = get_largest_army(state) + + has_longest_road = longest_road_color == p0_color + has_largest_army = largest_army_color == p0_color + + # Reward for VP gain + vp_gain = current_vp - self.prev_vp + reward += vp_gain * 1.0 + + # Reward for acquiring longest road + if has_longest_road and not self.prev_has_longest_road: + reward += 0.5 + elif not has_longest_road and self.prev_has_longest_road: + reward -= 0.5 # Lost longest road + + # Reward for acquiring largest army + if has_largest_army and not self.prev_has_largest_army: + reward += 0.5 + elif not has_largest_army and self.prev_has_largest_army: + reward -= 0.5 # Lost largest army + + # Check for game end + winning_color = game.winning_color() + if winning_color is not None: + if p0_color == winning_color: + reward += 10.0 # Big bonus for winning + else: + reward -= 10.0 # Penalty for losing + # Reset for next game + self.reset() + else: + # Update tracked state for next step + self.prev_vp = current_vp + self.prev_has_longest_road = has_longest_road + self.prev_has_largest_army = has_largest_army + + return reward + + +# Create a singleton instance to use +shaped_reward = ShapedRewardFunction() diff --git a/examples/ppo/train.py b/examples/ppo/train.py new file mode 100644 index 00000000..63f7277e --- /dev/null +++ b/examples/ppo/train.py @@ -0,0 +1,171 @@ +""" +Restartable Stable Baselines3 training example with TensorBoard logging. + +Features: +- Shaped reward function (incremental rewards for progress) +- GPU support (automatic detection) +- Checkpoint saving/loading for resumable training +- TensorBoard integration for monitoring + +Configure by editing constants: + NUM_LAYERS = 3 # Number of hidden layers + NEURONS_PER_LAYER = 256 # Neurons in each layer + USE_SHAPED_REWARD = True # Incremental vs sparse rewards + +Usage: + # Start new training: + python train.py + + # Resume from checkpoint: + python train.py --resume + + # View TensorBoard: + tensorboard --logdir ./tensorboard_logs +""" + +import argparse +import os +from pathlib import Path + +import gymnasium +import numpy as np +import torch +from sb3_contrib.common.maskable.policies import MaskableActorCriticPolicy +from sb3_contrib.common.wrappers import ActionMasker +from sb3_contrib.ppo_mask import MaskablePPO +from stable_baselines3.common.callbacks import CheckpointCallback + +import catanatron.gym +from catanatron import Color +from catanatron.players.value import ValueFunctionPlayer +from catanatron.gym.envs.catanatron_env import simple_reward +from shaped_reward import shaped_reward + + +# Configuration +SEED = 42 +CHECKPOINT_DIR = os.path.join(os.path.dirname(__file__), "checkpoints") +TENSORBOARD_DIR = os.path.join(os.path.dirname(__file__), "tensorboard_logs") +TOTAL_TIMESTEPS = 100_000 +CHECKPOINT_FREQ = 10_000 +NUM_LAYERS = 3 +NEURONS_PER_LAYER = 256 +USE_SHAPED_REWARD = True + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--resume", action="store_true", help="Resume from checkpoint") + args = parser.parse_args() + + # Seed everything for reproducibility + np.random.seed(SEED) + torch.manual_seed(SEED) + + # Set device (GPU if available) + device = ( + "cuda" + if torch.cuda.is_available() + else ("mps" if torch.backends.mps.is_available() else "cpu") + ) + print(f"Using device: {device}") + + # Create directories + os.makedirs(CHECKPOINT_DIR, exist_ok=True) + os.makedirs(TENSORBOARD_DIR, exist_ok=True) + + # Create environment + reward_fn = shaped_reward if USE_SHAPED_REWARD else simple_reward + print( + f"Using reward function: {'shaped (incremental)' if USE_SHAPED_REWARD else 'simple (sparse)'}" + ) + env = gymnasium.make( + "catanatron/Catanatron-v0", + config={ + "enemies": [ValueFunctionPlayer(Color.RED)], + "reward_function": reward_fn, + }, + ) + env = ActionMasker(env, mask_fn) + env.reset(seed=SEED) + + # Load or create model + if args.resume: + checkpoint = get_latest_checkpoint() + if checkpoint: + print(f"Loading checkpoint: {checkpoint}") + model = MaskablePPO.load( + checkpoint, + env=env, + tensorboard_log=TENSORBOARD_DIR, + device=device, + ) + else: + print("No checkpoint found, starting fresh") + args.resume = False + + if not args.resume: + # Configure network architecture + net_arch = [NEURONS_PER_LAYER] * NUM_LAYERS + policy_kwargs = dict(net_arch=net_arch) + + print(f"Creating new model with architecture: {net_arch}") + model = MaskablePPO( + MaskableActorCriticPolicy, + env, + verbose=1, + tensorboard_log=TENSORBOARD_DIR, + seed=SEED, + policy_kwargs=policy_kwargs, + device=device, + ) + + # Setup checkpoint callback + checkpoint_callback = CheckpointCallback( + save_freq=CHECKPOINT_FREQ, + save_path=CHECKPOINT_DIR, + name_prefix="rl_model", + ) + + # Train + print(f"\nTraining for {TOTAL_TIMESTEPS:,} timesteps") + print(f"TensorBoard: tensorboard --logdir {TENSORBOARD_DIR}\n") + + model.learn( + total_timesteps=TOTAL_TIMESTEPS, + callback=checkpoint_callback, + reset_num_timesteps=not args.resume, + tb_log_name="MaskablePPO", + ) + + # Save final model + final_path = os.path.join(CHECKPOINT_DIR, "final_model.zip") + model.save(final_path) + print(f"\nDone! Final model: {final_path}") + + +def mask_fn(env) -> np.ndarray: + """Create action mask for valid actions.""" + valid_actions = env.unwrapped.get_valid_actions() + mask = np.zeros(env.action_space.n, dtype=np.float32) + mask[valid_actions] = 1 + return np.array([bool(i) for i in mask]) + + +def get_latest_checkpoint(): + """Find the most recent checkpoint.""" + checkpoint_path = Path(CHECKPOINT_DIR) + if not checkpoint_path.exists(): + return None + + checkpoints = list(checkpoint_path.glob("rl_model_*_steps.zip")) + if not checkpoints: + return None + + # Get latest by timestep number + latest = max(checkpoints, key=lambda p: int(p.stem.split("_")[2])) + return str(latest) + + +if __name__ == "__main__": + main() From e3bdad32028a0b911ed6b3c6513c892bbc11dda4 Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Tue, 16 Dec 2025 21:47:33 -0400 Subject: [PATCH 02/19] Add BatchSize and N_STEPS --- examples/ppo/train.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/examples/ppo/train.py b/examples/ppo/train.py index 63f7277e..40206898 100644 --- a/examples/ppo/train.py +++ b/examples/ppo/train.py @@ -10,6 +10,8 @@ Configure by editing constants: NUM_LAYERS = 3 # Number of hidden layers NEURONS_PER_LAYER = 256 # Neurons in each layer + N_STEPS = 2048 # PPO rollout steps + BATCH_SIZE = 64 # PPO batch size USE_SHAPED_REWARD = True # Incremental vs sparse rewards Usage: @@ -52,6 +54,10 @@ NEURONS_PER_LAYER = 256 USE_SHAPED_REWARD = True +# PPO parameters +N_STEPS = 2048 # Number of steps to collect before update (default: 2048) +BATCH_SIZE = 128 # Batch size for training (default: 64) + def main(): parser = argparse.ArgumentParser() @@ -110,9 +116,12 @@ def main(): policy_kwargs = dict(net_arch=net_arch) print(f"Creating new model with architecture: {net_arch}") + print(f"PPO config: n_steps={N_STEPS}, batch_size={BATCH_SIZE}") model = MaskablePPO( MaskableActorCriticPolicy, env, + n_steps=N_STEPS, + batch_size=BATCH_SIZE, verbose=1, tensorboard_log=TENSORBOARD_DIR, seed=SEED, From 6f77d7f2d0b7e79a4d5d1f761c00c7a425861ee6 Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Thu, 18 Dec 2025 08:39:38 -0400 Subject: [PATCH 03/19] Add --timesteps --- README.md | 57 ++++++++++++++++++++++++------------------- examples/ppo/train.py | 20 +++++++++++---- 2 files changed, 47 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index d000b754..0e177e43 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,14 @@ ![Discord](https://img.shields.io/discord/1385302652014825552) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/bcollazo/catanatron/blob/master/examples/Overview.ipynb) -Catanatron is a high-performance simulator and strong AI player for Settlers of Catan. You can run thousands of games in the order of seconds. The goal is to find the strongest Settlers of Catan bot possible. +Catanatron is a high-performance simulator and strong AI player for Settlers of Catan. You can run thousands of games in the order of seconds. The goal is to find the strongest Settlers of Catan bot possible. Get Started with the Full Documentation: https://docs.catanatron.com Join our Discord: https://discord.gg/FgFmb75TWd! ## Command Line Interface + Catanatron provides a `catanatron-play` CLI tool to run large scale simulations.

@@ -22,26 +23,30 @@ Catanatron provides a `catanatron-play` CLI tool to run large scale simulations. 1. Clone the repository: - ```bash - git clone git@github.com:bcollazo/catanatron.git - cd catanatron/ - ``` -2. Create a virtual environment (requires Python 3.11 or higher) + ```bash + git clone git@github.com:bcollazo/catanatron.git + cd catanatron/ + ``` + +2. Create a virtual environment (requires Python 3.11 or higher) + + ```bash + python -m venv venv + source ./venv/bin/activate + # ./venv/Scripts/Activate.ps1 (on windows) + ``` - ```bash - python -m venv venv - source ./venv/bin/activate - ``` 3. Install dependencies - ```bash - pip install -e . - ``` -4. (Optional) Install developer and advanced dependencies + ```bash + pip install -e . + ``` + +4. (Optional) Install developer and advanced dependencies - ```bash - pip install -e ".[web,gym,dev]" - ``` + ```bash + pip install -e ".[web,gym,dev]" + ``` ### Usage @@ -52,13 +57,13 @@ catanatron-play --players=R,R,R,W --num=100 ``` Generate datasets from the games to analyze: + ```bash catanatron-play --num 100 --output my-data-path/ --output-format json ``` See more examples at https://docs.catanatron.com. - ## Graphical User Interface We provide Docker images so that you can watch, inspect, and play games against Catanatron via a web UI! @@ -67,15 +72,15 @@ We provide Docker images so that you can watch, inspect, and play games against

- ### Installation 1. Ensure you have Docker installed (https://docs.docker.com/engine/install/) 2. Run the `docker-compose.yaml` in the root folder of the repo: - ```bash - docker compose up - ``` + ```bash + docker compose up + ``` + 3. Visit http://localhost:3000 in your browser! ## Python Library @@ -100,14 +105,17 @@ print(game.play()) # returns winning color See more at http://docs.catanatron.com ## Gymnasium Interface + For Reinforcement Learning, catanatron provides an Open AI / Gymnasium Environment. Install it with: + ```bash pip install -e .[gym] ``` and use it like: + ```python import random import gymnasium @@ -128,8 +136,8 @@ env.close() See more at: https://docs.catanatron.com - ## Documentation + Full documentation here: https://docs.catanatron.com ## Contributing @@ -144,6 +152,5 @@ coverage run --source=catanatron -m pytest tests/ && coverage report See more at: https://docs.catanatron.com ## Appendix -See the motivation of the project here: [5 Ways NOT to Build a Catan AI](https://medium.com/@bcollazo2010/5-ways-not-to-build-a-catan-ai-e01bc491af17). - +See the motivation of the project here: [5 Ways NOT to Build a Catan AI](https://medium.com/@bcollazo2010/5-ways-not-to-build-a-catan-ai-e01bc491af17). diff --git a/examples/ppo/train.py b/examples/ppo/train.py index 40206898..c2c7e979 100644 --- a/examples/ppo/train.py +++ b/examples/ppo/train.py @@ -21,6 +21,9 @@ # Resume from checkpoint: python train.py --resume + # Train for custom timesteps: + python train.py --timesteps 500000 + # View TensorBoard: tensorboard --logdir ./tensorboard_logs """ @@ -48,20 +51,27 @@ SEED = 42 CHECKPOINT_DIR = os.path.join(os.path.dirname(__file__), "checkpoints") TENSORBOARD_DIR = os.path.join(os.path.dirname(__file__), "tensorboard_logs") -TOTAL_TIMESTEPS = 100_000 CHECKPOINT_FREQ = 10_000 NUM_LAYERS = 3 NEURONS_PER_LAYER = 256 USE_SHAPED_REWARD = True # PPO parameters -N_STEPS = 2048 # Number of steps to collect before update (default: 2048) -BATCH_SIZE = 128 # Batch size for training (default: 64) +N_ENVS = 1 # We might add Vectorized Environments later +N_STEPS = 8192 # Number of steps to collect before update (default: 2048) +BATCH_SIZE = 512 # Batch size for training (default: 64) +assert (N_ENVS * N_STEPS) % BATCH_SIZE == 0, "BATCH_SIZE must divide N_ENVS * N_STEPS" def main(): parser = argparse.ArgumentParser() parser.add_argument("--resume", action="store_true", help="Resume from checkpoint") + parser.add_argument( + "--timesteps", + type=int, + default=100_000, + help=f"Number of timesteps to train for (default: {100_000:,})", + ) args = parser.parse_args() # Seed everything for reproducibility @@ -137,11 +147,11 @@ def main(): ) # Train - print(f"\nTraining for {TOTAL_TIMESTEPS:,} timesteps") + print(f"\nTraining for {args.timesteps:,} timesteps") print(f"TensorBoard: tensorboard --logdir {TENSORBOARD_DIR}\n") model.learn( - total_timesteps=TOTAL_TIMESTEPS, + total_timesteps=args.timesteps, callback=checkpoint_callback, reset_num_timesteps=not args.resume, tb_log_name="MaskablePPO", From 78eefe0597b3127ac12fd535a2983e47d42afbd9 Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Fri, 19 Dec 2025 11:07:48 -0400 Subject: [PATCH 04/19] Entropy Coeff and Towards Reproducibility --- examples/ppo/train.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/examples/ppo/train.py b/examples/ppo/train.py index c2c7e979..a2864995 100644 --- a/examples/ppo/train.py +++ b/examples/ppo/train.py @@ -12,6 +12,7 @@ NEURONS_PER_LAYER = 256 # Neurons in each layer N_STEPS = 2048 # PPO rollout steps BATCH_SIZE = 64 # PPO batch size + ENT_COEF = 0.01 # Exploration (higher = more exploration) USE_SHAPED_REWARD = True # Incremental vs sparse rewards Usage: @@ -28,6 +29,7 @@ tensorboard --logdir ./tensorboard_logs """ +import random import argparse import os from pathlib import Path @@ -57,6 +59,7 @@ USE_SHAPED_REWARD = True # PPO parameters +ENT_COEF = 0.01 # Entropy coefficient for exploration (default: 0.0, try 0.01-0.1 for more exploration) N_ENVS = 1 # We might add Vectorized Environments later N_STEPS = 8192 # Number of steps to collect before update (default: 2048) BATCH_SIZE = 512 # Batch size for training (default: 64) @@ -74,9 +77,13 @@ def main(): ) args = parser.parse_args() - # Seed everything for reproducibility + # Set random seeds for reproducibility + random.seed(SEED) np.random.seed(SEED) torch.manual_seed(SEED) + torch.cuda.manual_seed_all(SEED) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False # Set device (GPU if available) device = ( @@ -126,12 +133,15 @@ def main(): policy_kwargs = dict(net_arch=net_arch) print(f"Creating new model with architecture: {net_arch}") - print(f"PPO config: n_steps={N_STEPS}, batch_size={BATCH_SIZE}") + print( + f"PPO config: n_steps={N_STEPS}, batch_size={BATCH_SIZE}, ent_coef={ENT_COEF}" + ) model = MaskablePPO( MaskableActorCriticPolicy, env, n_steps=N_STEPS, batch_size=BATCH_SIZE, + ent_coef=ENT_COEF, verbose=1, tensorboard_log=TENSORBOARD_DIR, seed=SEED, From fc01a0c53662b29d460e570f2f72db93492bf77f Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Fri, 19 Dec 2025 11:07:58 -0400 Subject: [PATCH 05/19] Refactor action_space utils to its own file --- .../catanatron/gym/envs/action_space.py | 91 +++++++++++++++ .../catanatron/gym/envs/catanatron_env.py | 109 +++--------------- .../machine_learning/players/reinforcement.py | 25 ++-- 3 files changed, 116 insertions(+), 109 deletions(-) create mode 100644 catanatron/catanatron/gym/envs/action_space.py diff --git a/catanatron/catanatron/gym/envs/action_space.py b/catanatron/catanatron/gym/envs/action_space.py new file mode 100644 index 00000000..a73db9a0 --- /dev/null +++ b/catanatron/catanatron/gym/envs/action_space.py @@ -0,0 +1,91 @@ +from catanatron.models.actions import Action +from catanatron.models.map import BASE_MAP_TEMPLATE, NUM_NODES, LandTile +from catanatron.models.enums import RESOURCES, ActionType +from catanatron.models.board import get_edges + +BASE_TOPOLOGY = BASE_MAP_TEMPLATE.topology +TILE_COORDINATES = [x for x, y in BASE_TOPOLOGY.items() if y == LandTile] +ACTIONS_ARRAY = [ + (ActionType.ROLL, None), + # TODO: One for each tile (and abuse 1v1 setting). + *[(ActionType.MOVE_ROBBER, tile) for tile in TILE_COORDINATES], + (ActionType.DISCARD, None), + *[(ActionType.BUILD_ROAD, tuple(sorted(edge))) for edge in get_edges()], + *[(ActionType.BUILD_SETTLEMENT, node_id) for node_id in range(NUM_NODES)], + *[(ActionType.BUILD_CITY, node_id) for node_id in range(NUM_NODES)], + (ActionType.BUY_DEVELOPMENT_CARD, None), + (ActionType.PLAY_KNIGHT_CARD, None), + *[ + (ActionType.PLAY_YEAR_OF_PLENTY, (first_card, RESOURCES[j])) + for i, first_card in enumerate(RESOURCES) + for j in range(i, len(RESOURCES)) + ], + *[(ActionType.PLAY_YEAR_OF_PLENTY, (first_card,)) for first_card in RESOURCES], + (ActionType.PLAY_ROAD_BUILDING, None), + *[(ActionType.PLAY_MONOPOLY, r) for r in RESOURCES], + # 4:1 with bank + *[ + (ActionType.MARITIME_TRADE, tuple(4 * [i] + [j])) + for i in RESOURCES + for j in RESOURCES + if i != j + ], + # 3:1 with port + *[ + (ActionType.MARITIME_TRADE, tuple(3 * [i] + [None, j])) # type: ignore + for i in RESOURCES + for j in RESOURCES + if i != j + ], + # 2:1 with port + *[ + (ActionType.MARITIME_TRADE, tuple(2 * [i] + [None, None, j])) # type: ignore + for i in RESOURCES + for j in RESOURCES + if i != j + ], + (ActionType.END_TURN, None), +] +ACTION_SPACE_SIZE = len(ACTIONS_ARRAY) +ACTION_TYPES = [i for i in ActionType] + + +def to_action_type_space(action_type: ActionType) -> int: + return ACTION_TYPES.index(action_type) + + +# NOTE: I think I don't need this if we separate action and action_record nicely... +def normalize_action(action): + normalized = action + if normalized.action_type == ActionType.ROLL: + return Action(action.color, action.action_type, None) + elif normalized.action_type == ActionType.MOVE_ROBBER: + return Action(action.color, action.action_type, action.value[0]) + elif normalized.action_type == ActionType.BUILD_ROAD: + return Action(action.color, action.action_type, tuple(sorted(action.value))) + elif normalized.action_type == ActionType.BUY_DEVELOPMENT_CARD: + return Action(action.color, action.action_type, None) + elif normalized.action_type == ActionType.DISCARD: + return Action(action.color, action.action_type, None) + return normalized + + +def to_action_space(action): + """maps action to space_action equivalent integer""" + normalized = normalize_action(action) + return ACTIONS_ARRAY.index((normalized.action_type, normalized.value)) + + +def from_action_space(action_int, playable_actions): + """maps action_int to catantron.models.actions.Action""" + # Get "catan_action" based on space action. + # i.e. Take first action in playable that matches ACTIONS_ARRAY blueprint + (action_type, value) = ACTIONS_ARRAY[action_int] + catan_action = None + for action in playable_actions: + normalized = normalize_action(action) + if normalized.action_type == action_type and normalized.value == value: + catan_action = action + break # return the first one + assert catan_action is not None + return catan_action diff --git a/catanatron/catanatron/gym/envs/catanatron_env.py b/catanatron/catanatron/gym/envs/catanatron_env.py index 623978b6..4fcb8500 100644 --- a/catanatron/catanatron/gym/envs/catanatron_env.py +++ b/catanatron/catanatron/gym/envs/catanatron_env.py @@ -5,13 +5,17 @@ from catanatron.game import Game, TURNS_LIMIT from catanatron.models.player import Color, Player, RandomPlayer -from catanatron.models.map import BASE_MAP_TEMPLATE, NUM_NODES, LandTile, build_map -from catanatron.models.enums import RESOURCES, Action, ActionType -from catanatron.models.board import get_edges +from catanatron.models.map import build_map from catanatron.features import ( create_sample, get_feature_ordering, ) +from catanatron.gym.action_space import ( + to_action_space, + from_action_space, + ACTION_SPACE_SIZE, + ACTIONS_ARRAY, +) from catanatron.gym.board_tensor_features import ( create_board_tensor, get_channels, @@ -19,94 +23,6 @@ ) -BASE_TOPOLOGY = BASE_MAP_TEMPLATE.topology -TILE_COORDINATES = [x for x, y in BASE_TOPOLOGY.items() if y == LandTile] -ACTIONS_ARRAY = [ - (ActionType.ROLL, None), - # TODO: One for each tile (and abuse 1v1 setting). - *[(ActionType.MOVE_ROBBER, tile) for tile in TILE_COORDINATES], - (ActionType.DISCARD, None), - *[(ActionType.BUILD_ROAD, tuple(sorted(edge))) for edge in get_edges()], - *[(ActionType.BUILD_SETTLEMENT, node_id) for node_id in range(NUM_NODES)], - *[(ActionType.BUILD_CITY, node_id) for node_id in range(NUM_NODES)], - (ActionType.BUY_DEVELOPMENT_CARD, None), - (ActionType.PLAY_KNIGHT_CARD, None), - *[ - (ActionType.PLAY_YEAR_OF_PLENTY, (first_card, RESOURCES[j])) - for i, first_card in enumerate(RESOURCES) - for j in range(i, len(RESOURCES)) - ], - *[(ActionType.PLAY_YEAR_OF_PLENTY, (first_card,)) for first_card in RESOURCES], - (ActionType.PLAY_ROAD_BUILDING, None), - *[(ActionType.PLAY_MONOPOLY, r) for r in RESOURCES], - # 4:1 with bank - *[ - (ActionType.MARITIME_TRADE, tuple(4 * [i] + [j])) - for i in RESOURCES - for j in RESOURCES - if i != j - ], - # 3:1 with port - *[ - (ActionType.MARITIME_TRADE, tuple(3 * [i] + [None, j])) # type: ignore - for i in RESOURCES - for j in RESOURCES - if i != j - ], - # 2:1 with port - *[ - (ActionType.MARITIME_TRADE, tuple(2 * [i] + [None, None, j])) # type: ignore - for i in RESOURCES - for j in RESOURCES - if i != j - ], - (ActionType.END_TURN, None), -] -ACTION_SPACE_SIZE = len(ACTIONS_ARRAY) -ACTION_TYPES = [i for i in ActionType] - - -def to_action_type_space(action_type: ActionType) -> int: - return ACTION_TYPES.index(action_type) - - -# NOTE: I think I don't need this if we separate action and action_record nicely... -def normalize_action(action): - normalized = action - if normalized.action_type == ActionType.ROLL: - return Action(action.color, action.action_type, None) - elif normalized.action_type == ActionType.MOVE_ROBBER: - return Action(action.color, action.action_type, action.value[0]) - elif normalized.action_type == ActionType.BUILD_ROAD: - return Action(action.color, action.action_type, tuple(sorted(action.value))) - elif normalized.action_type == ActionType.BUY_DEVELOPMENT_CARD: - return Action(action.color, action.action_type, None) - elif normalized.action_type == ActionType.DISCARD: - return Action(action.color, action.action_type, None) - return normalized - - -def to_action_space(action): - """maps action to space_action equivalent integer""" - normalized = normalize_action(action) - return ACTIONS_ARRAY.index((normalized.action_type, normalized.value)) - - -def from_action_space(action_int, playable_actions): - """maps action_int to catantron.models.actions.Action""" - # Get "catan_action" based on space action. - # i.e. Take first action in playable that matches ACTIONS_ARRAY blueprint - (action_type, value) = ACTIONS_ARRAY[action_int] - catan_action = None - for action in playable_actions: - normalized = normalize_action(action) - if normalized.action_type == action_type and normalized.value == value: - catan_action = action - break # return the first one - assert catan_action is not None - return catan_action - - FEATURES = get_feature_ordering(num_players=2) NUM_FEATURES = len(FEATURES) @@ -188,6 +104,17 @@ def get_valid_actions(self): """ return list(map(to_action_space, self.game.playable_actions)) + def action_masks(self) -> list[bool]: + """ + This method is to be compatible with SB3 SubprocVecEnv. + See https://sb3-contrib.readthedocs.io/en/master/modules/ppo_mask.html + + Returns: + List[bool]: action masks + """ + valid = set(self.get_valid_actions()) + return [action_int in valid for action_int in range(ACTION_SPACE_SIZE)] + def step(self, action): try: catan_action = from_action_space(action, self.game.playable_actions) diff --git a/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py b/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py index 705d52ac..c863e482 100644 --- a/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py +++ b/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py @@ -5,13 +5,16 @@ from tensorflow import keras from catanatron.models.player import Player -from catanatron.models.enums import Action, ActionType from catanatron.features import ( create_sample, create_sample_vector, get_feature_ordering, ) -from catanatron.gym.envs.catanatron_env import ACTIONS_ARRAY, ACTION_SPACE_SIZE +from catanatron.gym.envs.action_space import ( + ACTIONS_ARRAY, + ACTION_SPACE_SIZE, + normalize_action, +) from catanatron.gym.board_tensor_features import ( NUMERIC_FEATURES, create_board_tensor, @@ -94,6 +97,8 @@ def p_model_path(version): def get_v_model(model_path): global V_MODEL if V_MODEL is None: + import autokeras as ak + custom_objects = None if model_path[:2] != "ak" else ak.CUSTOM_OBJECTS V_MODEL = keras.models.load_model(model_path, custom_objects=custom_objects) return V_MODEL @@ -115,22 +120,6 @@ def hot_one_encode_action(action): return vector -def normalize_action(action): - normalized = action - if normalized.action_type == ActionType.ROLL: - return Action(action.color, action.action_type, None) - elif normalized.action_type == ActionType.MOVE_ROBBER: - return Action(action.color, action.action_type, action.value[0]) - elif normalized.action_type == ActionType.BUILD_ROAD: - return Action(action.color, action.action_type, tuple(sorted(action.value))) - elif normalized.action_type == ActionType.BUY_DEVELOPMENT_CARD: - return Action(action.color, action.action_type, None) - elif normalized.action_type == ActionType.DISCARD: - return Action(action.color, action.action_type, None) - - return normalized - - class PRLPlayer(Player): def __init__(self, color, model_path): super(PRLPlayer, self).__init__(color) From b7de6d8042d6ea7350f1cb6b649cb9b3ac4e402d Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Fri, 19 Dec 2025 11:27:40 -0400 Subject: [PATCH 06/19] Change ActionSpace to not need normalization (and changing numbers around) --- catanatron/catanatron/gym/accumulators.py | 2 +- .../catanatron/gym/envs/action_space.py | 32 ++++++------------- .../catanatron/gym/envs/catanatron_env.py | 2 +- catanatron/catanatron/models/enums.py | 4 ++- .../machine_learning/players/reinforcement.py | 7 ++-- 5 files changed, 16 insertions(+), 31 deletions(-) diff --git a/catanatron/catanatron/gym/accumulators.py b/catanatron/catanatron/gym/accumulators.py index fd9e6321..fc4d6808 100644 --- a/catanatron/catanatron/gym/accumulators.py +++ b/catanatron/catanatron/gym/accumulators.py @@ -9,7 +9,7 @@ from catanatron.features import create_sample from catanatron.game import GameAccumulator from catanatron.gym.board_tensor_features import create_board_tensor -from catanatron.gym.envs.catanatron_env import to_action_space, to_action_type_space +from catanatron.gym.envs.action_space import to_action_space, to_action_type_space from catanatron.gym.utils import ( DISCOUNT_FACTOR, get_tournament_total_return, diff --git a/catanatron/catanatron/gym/envs/action_space.py b/catanatron/catanatron/gym/envs/action_space.py index a73db9a0..2c3e3497 100644 --- a/catanatron/catanatron/gym/envs/action_space.py +++ b/catanatron/catanatron/gym/envs/action_space.py @@ -1,14 +1,12 @@ -from catanatron.models.actions import Action from catanatron.models.map import BASE_MAP_TEMPLATE, NUM_NODES, LandTile from catanatron.models.enums import RESOURCES, ActionType from catanatron.models.board import get_edges +from catanatron.models.player import Color BASE_TOPOLOGY = BASE_MAP_TEMPLATE.topology TILE_COORDINATES = [x for x, y in BASE_TOPOLOGY.items() if y == LandTile] ACTIONS_ARRAY = [ (ActionType.ROLL, None), - # TODO: One for each tile (and abuse 1v1 setting). - *[(ActionType.MOVE_ROBBER, tile) for tile in TILE_COORDINATES], (ActionType.DISCARD, None), *[(ActionType.BUILD_ROAD, tuple(sorted(edge))) for edge in get_edges()], *[(ActionType.BUILD_SETTLEMENT, node_id) for node_id in range(NUM_NODES)], @@ -23,6 +21,12 @@ *[(ActionType.PLAY_YEAR_OF_PLENTY, (first_card,)) for first_card in RESOURCES], (ActionType.PLAY_ROAD_BUILDING, None), *[(ActionType.PLAY_MONOPOLY, r) for r in RESOURCES], + # Move Robber actions include to every tile and from each opponent + *[ + (ActionType.MOVE_ROBBER, (tile, victim_color)) + for tile in TILE_COORDINATES + for victim_color in [None] + [color for color in Color] + ], # 4:1 with bank *[ (ActionType.MARITIME_TRADE, tuple(4 * [i] + [j])) @@ -54,26 +58,9 @@ def to_action_type_space(action_type: ActionType) -> int: return ACTION_TYPES.index(action_type) -# NOTE: I think I don't need this if we separate action and action_record nicely... -def normalize_action(action): - normalized = action - if normalized.action_type == ActionType.ROLL: - return Action(action.color, action.action_type, None) - elif normalized.action_type == ActionType.MOVE_ROBBER: - return Action(action.color, action.action_type, action.value[0]) - elif normalized.action_type == ActionType.BUILD_ROAD: - return Action(action.color, action.action_type, tuple(sorted(action.value))) - elif normalized.action_type == ActionType.BUY_DEVELOPMENT_CARD: - return Action(action.color, action.action_type, None) - elif normalized.action_type == ActionType.DISCARD: - return Action(action.color, action.action_type, None) - return normalized - - def to_action_space(action): """maps action to space_action equivalent integer""" - normalized = normalize_action(action) - return ACTIONS_ARRAY.index((normalized.action_type, normalized.value)) + return ACTIONS_ARRAY.index((action.action_type, action.value)) def from_action_space(action_int, playable_actions): @@ -83,8 +70,7 @@ def from_action_space(action_int, playable_actions): (action_type, value) = ACTIONS_ARRAY[action_int] catan_action = None for action in playable_actions: - normalized = normalize_action(action) - if normalized.action_type == action_type and normalized.value == value: + if action.action_type == action_type and action.value == value: catan_action = action break # return the first one assert catan_action is not None diff --git a/catanatron/catanatron/gym/envs/catanatron_env.py b/catanatron/catanatron/gym/envs/catanatron_env.py index 4fcb8500..4160f57a 100644 --- a/catanatron/catanatron/gym/envs/catanatron_env.py +++ b/catanatron/catanatron/gym/envs/catanatron_env.py @@ -10,7 +10,7 @@ create_sample, get_feature_ordering, ) -from catanatron.gym.action_space import ( +from catanatron.gym.envs.action_space import ( to_action_space, from_action_space, ACTION_SPACE_SIZE, diff --git a/catanatron/catanatron/models/enums.py b/catanatron/catanatron/models/enums.py index 49c0debb..a129cd45 100644 --- a/catanatron/catanatron/models/enums.py +++ b/catanatron/catanatron/models/enums.py @@ -74,7 +74,9 @@ class ActionType(Enum): ROLL = "ROLL" # value is None MOVE_ROBBER = "MOVE_ROBBER" # value is (coordinate, Color|None). - DISCARD = "DISCARD" # value is None|Resource[]. TODO: Should always be Resource[]. + + # TODO: None for now to avoid complexity, but should be Resource[]. + DISCARD = "DISCARD" # value is None # Building/Buying BUILD_ROAD = "BUILD_ROAD" # value is edge_id diff --git a/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py b/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py index c863e482..a9688467 100644 --- a/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py +++ b/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py @@ -13,7 +13,6 @@ from catanatron.gym.envs.action_space import ( ACTIONS_ARRAY, ACTION_SPACE_SIZE, - normalize_action, ) from catanatron.gym.board_tensor_features import ( NUMERIC_FEATURES, @@ -113,8 +112,7 @@ def get_t_model(model_path): def hot_one_encode_action(action): - normalized = normalize_action(action) - index = ACTIONS_ARRAY.index((normalized.action_type, normalized.value)) + index = ACTIONS_ARRAY.index((action.action_type, action.value)) vector = np.zeros(ACTION_SPACE_SIZE, dtype=int) vector[index] = 1 return vector @@ -136,8 +134,7 @@ def decide(self, game, playable_actions): # return playable_actions[index] # Create array like [0,0,1,0,0,0,1,...] representing possible actions - normalized_playable = [normalize_action(a) for a in playable_actions] - possibilities = [(a.action_type, a.value) for a in normalized_playable] + possibilities = [(a.action_type, a.value) for a in playable_actions] possible_indices = [ACTIONS_ARRAY.index(x) for x in possibilities] mask = np.zeros(ACTION_SPACE_SIZE, dtype=np.int) mask[possible_indices] = 1 From ac4d33f1d2e7b17d3b2215f6085394470e92e623 Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Fri, 19 Dec 2025 19:39:56 -0400 Subject: [PATCH 07/19] Fix Ruff Warnings --- catanatron/catanatron/cli/play.py | 3 +-- catanatron/catanatron/features.py | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/catanatron/catanatron/cli/play.py b/catanatron/catanatron/cli/play.py index cf098f95..17e848a2 100644 --- a/catanatron/catanatron/cli/play.py +++ b/catanatron/catanatron/cli/play.py @@ -7,9 +7,8 @@ from rich.console import Console from rich.table import Table from rich.progress import Progress -from rich.progress import Progress, BarColumn, TimeRemainingColumn +from rich.progress import BarColumn, TimeRemainingColumn from rich import box -from rich.console import Console from rich.theme import Theme from rich.text import Text diff --git a/catanatron/catanatron/features.py b/catanatron/catanatron/features.py index d74e0f89..a09567c7 100644 --- a/catanatron/catanatron/features.py +++ b/catanatron/catanatron/features.py @@ -12,7 +12,7 @@ ) from catanatron.models.board import STATIC_GRAPH, get_edges, get_node_distances from catanatron.models.map import NUM_TILES, CatanMap, build_map, number_probability -from catanatron.models.player import Player, Color, SimplePlayer +from catanatron.models.player import Color, SimplePlayer from catanatron.models.enums import ( DEVELOPMENT_CARDS, RESOURCES, @@ -106,7 +106,7 @@ def resource_hand_features(game: Game, p0_color: Color): ] for card in DEVELOPMENT_CARDS: features[f"P0_{card}_IN_HAND"] = player_state[key + f"_{card}_IN_HAND"] - features[f"P0_HAS_PLAYED_DEVELOPMENT_CARD_IN_TURN"] = player_state[ + features["P0_HAS_PLAYED_DEVELOPMENT_CARD_IN_TURN"] = player_state[ key + "_HAS_PLAYED_DEVELOPMENT_CARD_IN_TURN" ] @@ -132,7 +132,7 @@ def map_tile_features(catan_map: CatanMap, robber_coordinate): for tile_id, tile in catan_map.tiles_by_id.items(): for resource in RESOURCES: features[f"TILE{tile_id}_IS_{resource}"] = tile.resource == resource - features[f"TILE{tile_id}_IS_DESERT"] = tile.resource == None + features[f"TILE{tile_id}_IS_DESERT"] = tile.resource is None features[f"TILE{tile_id}_PROBA"] = ( 0 if tile.resource is None else number_probability(tile.number) ) From 44aa80bc6572e94461766662e1ce3086e0a0a65c Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Fri, 19 Dec 2025 20:07:37 -0400 Subject: [PATCH 08/19] Fix ruff warning --- catanatron/catanatron/models/map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catanatron/catanatron/models/map.py b/catanatron/catanatron/models/map.py index 2219bac8..6d8c9a3f 100644 --- a/catanatron/catanatron/models/map.py +++ b/catanatron/catanatron/models/map.py @@ -367,7 +367,7 @@ def initialize_tiles( port_autoinc += 1 elif tile_type == LandTile: resource = shuffled_tile_resources.pop() - if resource != None: + if resource is not None: number = shuffled_numbers.pop() tile = LandTile(tile_autoinc, resource, number, nodes, edges) else: From dbaa9accbe4a7903fa297e9b719e91ab69653a72 Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Fri, 19 Dec 2025 20:08:52 -0400 Subject: [PATCH 09/19] Dynamic Action Space per Map Type (and fix test) --- catanatron/catanatron/cli/play.py | 12 +- catanatron/catanatron/gym/accumulators.py | 34 +++-- .../catanatron/gym/envs/action_space.py | 121 ++++++++++-------- .../catanatron/gym/envs/catanatron_env.py | 25 ++-- catanatron/catanatron/models/map.py | 9 ++ .../machine_learning/players/reinforcement.py | 7 +- tests/test_gym.py | 2 +- 7 files changed, 128 insertions(+), 82 deletions(-) diff --git a/catanatron/catanatron/cli/play.py b/catanatron/catanatron/cli/play.py index 17e848a2..00ebd5fc 100644 --- a/catanatron/catanatron/cli/play.py +++ b/catanatron/catanatron/cli/play.py @@ -206,7 +206,7 @@ class OutputOptions: class GameConfigOptions: discard_limit: int = 7 vps_to_win: int = 10 - catan_map: Literal["BASE", "TOURNAMENT", "MINI"] = "BASE" + map_type: Literal["BASE", "TOURNAMENT", "MINI"] = "BASE" COLOR_TO_RICH_STYLE = { @@ -237,7 +237,7 @@ def play_batch_core(num_games, players, game_config, accumulators=[]): for _ in range(num_games): for player in players: player.reset_state() - catan_map = build_map(game_config.catan_map) + catan_map = build_map(game_config.map_type) game = Game( players, discard_limit=game_config.discard_limit, @@ -274,7 +274,9 @@ def play_batch( accumulators.append( CsvDataAccumulator( - output_options.output, output_options.include_board_tensor + game_config.map_type, + output_options.output, + output_options.include_board_tensor, ) ) elif output_options.output_format == "parquet": @@ -283,7 +285,9 @@ def play_batch( accumulators.append( ParquetDataAccumulator( - output_options.output, output_options.include_board_tensor + game_config.map_type, + output_options.output, + output_options.include_board_tensor, ) ) elif output_options.output_format == "json": diff --git a/catanatron/catanatron/gym/accumulators.py b/catanatron/catanatron/gym/accumulators.py index fc4d6808..c6b23ab0 100644 --- a/catanatron/catanatron/gym/accumulators.py +++ b/catanatron/catanatron/gym/accumulators.py @@ -1,8 +1,8 @@ import os -from collections import defaultdict import time +from collections import defaultdict +from typing import Literal -from catanatron.utils import format_secs import numpy as np import pandas as pd @@ -17,11 +17,15 @@ populate_matrices, simple_total_return, ) +from catanatron.models.actions import Action +from catanatron.game import Game +from catanatron.utils import format_secs class ReinforcementLearningAccumulator(GameAccumulator): def __init__( self, + map_type: Literal["BASE", "TOURNAMENT", "MINI"] = "BASE", include_board_tensor=True, total_return_fns={ "RETURN": simple_total_return, @@ -29,6 +33,7 @@ def __init__( "VICTORY_POINTS_RETURN": get_victory_points_total_return, }, ): + self.map_type = map_type self.include_board_tensor = include_board_tensor # TODO: Generalize to "rewards_fn" that can yield intermediary rewards # while still rewarding big on terminal states. @@ -45,14 +50,17 @@ def before(self, game): if self.include_board_tensor: self.data["board_tensors"] = [] - def step(self, game_before_action, action): + def step(self, game_before_action: Game, action: Action): self.data["color_action_indices"][action.color].append( len(self.data["samples"]) ) self.data["acting_color"].append(action.color) self.data["samples"].append(create_sample(game_before_action, action.color)) self.data["actions"].append( - [to_action_space(action), to_action_type_space(action.action_type)] + [ + to_action_space(action, self.map_type), + to_action_type_space(action.action_type), + ] ) if self.include_board_tensor: @@ -130,8 +138,13 @@ def after(self, game): class CsvDataAccumulator(ReinforcementLearningAccumulator): - def __init__(self, output, include_board_tensor=True): - super().__init__(include_board_tensor) + def __init__( + self, + map_type: Literal["BASE", "TOURNAMENT", "MINI"], + output, + include_board_tensor=True, + ): + super().__init__(map_type, include_board_tensor) self.output = output def after(self, game): @@ -164,8 +177,13 @@ def after(self, game): class ParquetDataAccumulator(ReinforcementLearningAccumulator): - def __init__(self, output, include_board_tensor=True): - super().__init__(include_board_tensor) + def __init__( + self, + map_type: Literal["BASE", "TOURNAMENT", "MINI"], + output, + include_board_tensor=True, + ): + super().__init__(map_type, include_board_tensor) self.output = output def after(self, game): diff --git a/catanatron/catanatron/gym/envs/action_space.py b/catanatron/catanatron/gym/envs/action_space.py index 2c3e3497..c9b2ab53 100644 --- a/catanatron/catanatron/gym/envs/action_space.py +++ b/catanatron/catanatron/gym/envs/action_space.py @@ -1,56 +1,65 @@ -from catanatron.models.map import BASE_MAP_TEMPLATE, NUM_NODES, LandTile +from typing import Literal +from functools import lru_cache + +from catanatron.models.actions import Action +from catanatron.models.map import get_map_template, NUM_NODES, LandTile from catanatron.models.enums import RESOURCES, ActionType from catanatron.models.board import get_edges from catanatron.models.player import Color -BASE_TOPOLOGY = BASE_MAP_TEMPLATE.topology -TILE_COORDINATES = [x for x, y in BASE_TOPOLOGY.items() if y == LandTile] -ACTIONS_ARRAY = [ - (ActionType.ROLL, None), - (ActionType.DISCARD, None), - *[(ActionType.BUILD_ROAD, tuple(sorted(edge))) for edge in get_edges()], - *[(ActionType.BUILD_SETTLEMENT, node_id) for node_id in range(NUM_NODES)], - *[(ActionType.BUILD_CITY, node_id) for node_id in range(NUM_NODES)], - (ActionType.BUY_DEVELOPMENT_CARD, None), - (ActionType.PLAY_KNIGHT_CARD, None), - *[ - (ActionType.PLAY_YEAR_OF_PLENTY, (first_card, RESOURCES[j])) - for i, first_card in enumerate(RESOURCES) - for j in range(i, len(RESOURCES)) - ], - *[(ActionType.PLAY_YEAR_OF_PLENTY, (first_card,)) for first_card in RESOURCES], - (ActionType.PLAY_ROAD_BUILDING, None), - *[(ActionType.PLAY_MONOPOLY, r) for r in RESOURCES], - # Move Robber actions include to every tile and from each opponent - *[ - (ActionType.MOVE_ROBBER, (tile, victim_color)) - for tile in TILE_COORDINATES - for victim_color in [None] + [color for color in Color] - ], - # 4:1 with bank - *[ - (ActionType.MARITIME_TRADE, tuple(4 * [i] + [j])) - for i in RESOURCES - for j in RESOURCES - if i != j - ], - # 3:1 with port - *[ - (ActionType.MARITIME_TRADE, tuple(3 * [i] + [None, j])) # type: ignore - for i in RESOURCES - for j in RESOURCES - if i != j - ], - # 2:1 with port - *[ - (ActionType.MARITIME_TRADE, tuple(2 * [i] + [None, None, j])) # type: ignore - for i in RESOURCES - for j in RESOURCES - if i != j - ], - (ActionType.END_TURN, None), -] -ACTION_SPACE_SIZE = len(ACTIONS_ARRAY) + +@lru_cache(maxsize=None) +def get_action_array(map_type: Literal["BASE", "TOURNAMENT", "MINI"]): + map_template = get_map_template(map_type) + tile_coordinates = [x for x, y in map_template.topology.items() if y == LandTile] + actions_array = [ + (ActionType.ROLL, None), + (ActionType.DISCARD, None), + *[(ActionType.BUILD_ROAD, tuple(sorted(edge))) for edge in get_edges()], + *[(ActionType.BUILD_SETTLEMENT, node_id) for node_id in range(NUM_NODES)], + *[(ActionType.BUILD_CITY, node_id) for node_id in range(NUM_NODES)], + (ActionType.BUY_DEVELOPMENT_CARD, None), + (ActionType.PLAY_KNIGHT_CARD, None), + *[ + (ActionType.PLAY_YEAR_OF_PLENTY, (first_card, RESOURCES[j])) + for i, first_card in enumerate(RESOURCES) + for j in range(i, len(RESOURCES)) + ], + *[(ActionType.PLAY_YEAR_OF_PLENTY, (first_card,)) for first_card in RESOURCES], + (ActionType.PLAY_ROAD_BUILDING, None), + *[(ActionType.PLAY_MONOPOLY, r) for r in RESOURCES], + # Move Robber actions include to every tile and from each opponent + *[ + (ActionType.MOVE_ROBBER, (tile, victim_color)) + for tile in tile_coordinates + for victim_color in [None] + [color for color in Color] + ], + # 4:1 with bank + *[ + (ActionType.MARITIME_TRADE, tuple(4 * [i] + [j])) + for i in RESOURCES + for j in RESOURCES + if i != j + ], + # 3:1 with port + *[ + (ActionType.MARITIME_TRADE, tuple(3 * [i] + [None, j])) # type: ignore + for i in RESOURCES + for j in RESOURCES + if i != j + ], + # 2:1 with port + *[ + (ActionType.MARITIME_TRADE, tuple(2 * [i] + [None, None, j])) # type: ignore + for i in RESOURCES + for j in RESOURCES + if i != j + ], + (ActionType.END_TURN, None), + ] + return actions_array + + ACTION_TYPES = [i for i in ActionType] @@ -58,16 +67,20 @@ def to_action_type_space(action_type: ActionType) -> int: return ACTION_TYPES.index(action_type) -def to_action_space(action): +def to_action_space(action: Action, map_type: Literal["BASE", "TOURNAMENT", "MINI"]): """maps action to space_action equivalent integer""" - return ACTIONS_ARRAY.index((action.action_type, action.value)) + actions_array = get_action_array(map_type) + return actions_array.index((action.action_type, action.value)) -def from_action_space(action_int, playable_actions): +def from_action_space( + action_int, playable_actions, map_type: Literal["BASE", "TOURNAMENT", "MINI"] +): """maps action_int to catantron.models.actions.Action""" # Get "catan_action" based on space action. - # i.e. Take first action in playable that matches ACTIONS_ARRAY blueprint - (action_type, value) = ACTIONS_ARRAY[action_int] + # i.e. Take first action in playable that matches actions_array blueprint + actions_array = get_action_array(map_type) + (action_type, value) = actions_array[action_int] catan_action = None for action in playable_actions: if action.action_type == action_type and action.value == value: diff --git a/catanatron/catanatron/gym/envs/catanatron_env.py b/catanatron/catanatron/gym/envs/catanatron_env.py index 4160f57a..bb1bcea9 100644 --- a/catanatron/catanatron/gym/envs/catanatron_env.py +++ b/catanatron/catanatron/gym/envs/catanatron_env.py @@ -13,8 +13,7 @@ from catanatron.gym.envs.action_space import ( to_action_space, from_action_space, - ACTION_SPACE_SIZE, - ACTIONS_ARRAY, + get_action_array, ) from catanatron.gym.board_tensor_features import ( create_board_tensor, @@ -67,8 +66,10 @@ def __init__(self, config=None): self.invalid_actions_count = 0 self.max_invalid_actions = 10 - # TODO: Make self.action_space tighter if possible (per map_type) - self.action_space = spaces.Discrete(ACTION_SPACE_SIZE) + # Build action space depending on map type + self.action_array = get_action_array(self.map_type) + self.action_space_size = len(self.action_array) + self.action_space = spaces.Discrete(self.action_space_size) if self.representation == "mixed": channels = get_channels(len(self.players)) @@ -102,7 +103,7 @@ def get_valid_actions(self): Returns: List[int]: valid actions """ - return list(map(to_action_space, self.game.playable_actions)) + return [to_action_space(a, self.map_type) for a in self.game.playable_actions] def action_masks(self) -> list[bool]: """ @@ -113,20 +114,18 @@ def action_masks(self) -> list[bool]: List[bool]: action masks """ valid = set(self.get_valid_actions()) - return [action_int in valid for action_int in range(ACTION_SPACE_SIZE)] + return [action_int in valid for action_int in range(self.action_space_size)] def step(self, action): try: - catan_action = from_action_space(action, self.game.playable_actions) - except Exception as e: + catan_action = from_action_space( + action, self.game.playable_actions, self.map_type + ) + except Exception: self.invalid_actions_count += 1 observation = self._get_observation() winning_color = self.game.winning_color() - done = ( - winning_color is not None - or self.invalid_actions_count > self.max_invalid_actions - ) terminated = winning_color is not None truncated = ( self.invalid_actions_count > self.max_invalid_actions @@ -225,7 +224,7 @@ def _advance_until_p0_decision(self): * - Integer - Catanatron Action """ -for i, v in enumerate(ACTIONS_ARRAY): +for i, v in enumerate(get_action_array("BASE")): CatanatronEnv.__doc__ += f" * - {i}\n - {v}\n" CatanatronEnv.__doc__ += """ diff --git a/catanatron/catanatron/models/map.py b/catanatron/catanatron/models/map.py index 6d8c9a3f..6835ce07 100644 --- a/catanatron/catanatron/models/map.py +++ b/catanatron/catanatron/models/map.py @@ -192,6 +192,15 @@ class MapTemplate: ) +def get_map_template(map_type: Literal["BASE", "TOURNAMENT", "MINI"]): + if map_type == "TOURNAMENT" or map_type == "BASE": + return BASE_MAP_TEMPLATE + elif map_type == "MINI": + return MINI_MAP_TEMPLATE + else: + raise ValueError(f"Invalid map_type: {map_type}") + + class CatanMap: """Represents a randomly initialized map.""" diff --git a/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py b/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py index a9688467..b92e9978 100644 --- a/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py +++ b/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py @@ -11,14 +11,17 @@ get_feature_ordering, ) from catanatron.gym.envs.action_space import ( - ACTIONS_ARRAY, - ACTION_SPACE_SIZE, + get_action_array, ) from catanatron.gym.board_tensor_features import ( NUMERIC_FEATURES, create_board_tensor, ) + +ACTIONS_ARRAY = get_action_array("BASE") +ACTION_SPACE_SIZE = len(ACTIONS_ARRAY) + # from catanatron_experimental.rep_b_model import build_model # Taken from correlation analysis diff --git a/tests/test_gym.py b/tests/test_gym.py index f7b5dfbe..e1e235ff 100644 --- a/tests/test_gym.py +++ b/tests/test_gym.py @@ -84,7 +84,7 @@ def test_invalid_action_reward(): def test_custom_reward(): - def custom_reward(game, p0_color): + def custom_reward(action, game, p0_color): return 123 env = gymnasium.make( From 8e22162236fc0f63080954d0ce4fe2df16124576 Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Fri, 19 Dec 2025 20:51:01 -0400 Subject: [PATCH 10/19] Use Vectorized Environment --- examples/ppo/train.py | 52 ++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/examples/ppo/train.py b/examples/ppo/train.py index a2864995..78d9a094 100644 --- a/examples/ppo/train.py +++ b/examples/ppo/train.py @@ -2,16 +2,18 @@ Restartable Stable Baselines3 training example with TensorBoard logging. Features: +- Vectorized environments (8 parallel games for ~8x speedup) - Shaped reward function (incremental rewards for progress) - GPU support (automatic detection) - Checkpoint saving/loading for resumable training - TensorBoard integration for monitoring Configure by editing constants: + N_ENVS = 8 # Parallel environments NUM_LAYERS = 3 # Number of hidden layers NEURONS_PER_LAYER = 256 # Neurons in each layer N_STEPS = 2048 # PPO rollout steps - BATCH_SIZE = 64 # PPO batch size + BATCH_SIZE = 256 # PPO batch size ENT_COEF = 0.01 # Exploration (higher = more exploration) USE_SHAPED_REWARD = True # Incremental vs sparse rewards @@ -41,12 +43,14 @@ from sb3_contrib.common.wrappers import ActionMasker from sb3_contrib.ppo_mask import MaskablePPO from stable_baselines3.common.callbacks import CheckpointCallback +from stable_baselines3.common.env_util import make_vec_env +from stable_baselines3.common.vec_env import SubprocVecEnv import catanatron.gym from catanatron import Color from catanatron.players.value import ValueFunctionPlayer from catanatron.gym.envs.catanatron_env import simple_reward -from shaped_reward import shaped_reward +from shaped_reward import ShapedRewardFunction # Configuration @@ -60,12 +64,28 @@ # PPO parameters ENT_COEF = 0.01 # Entropy coefficient for exploration (default: 0.0, try 0.01-0.1 for more exploration) -N_ENVS = 1 # We might add Vectorized Environments later -N_STEPS = 8192 # Number of steps to collect before update (default: 2048) -BATCH_SIZE = 512 # Batch size for training (default: 64) +N_ENVS = 1 # Number of parallel environments +N_STEPS = 2048 # Number of steps to collect before update (default: 2048) +BATCH_SIZE = 256 # Batch size for training (default: 64) assert (N_ENVS * N_STEPS) % BATCH_SIZE == 0, "BATCH_SIZE must divide N_ENVS * N_STEPS" +def make_catan_env(): + """Factory function to create a Catan environment for vectorization.""" + # Create fresh reward function instance for each environment + reward_fn = ShapedRewardFunction() if USE_SHAPED_REWARD else simple_reward + + env = gymnasium.make( + "catanatron/Catanatron-v0", + config={ + "enemies": [ValueFunctionPlayer(Color.RED)], + "reward_function": reward_fn, + }, + ) + env = ActionMasker(env, mask_fn) + return env + + def main(): parser = argparse.ArgumentParser() parser.add_argument("--resume", action="store_true", help="Resume from checkpoint") @@ -97,20 +117,18 @@ def main(): os.makedirs(CHECKPOINT_DIR, exist_ok=True) os.makedirs(TENSORBOARD_DIR, exist_ok=True) - # Create environment - reward_fn = shaped_reward if USE_SHAPED_REWARD else simple_reward + # Create vectorized environments print( f"Using reward function: {'shaped (incremental)' if USE_SHAPED_REWARD else 'simple (sparse)'}" ) - env = gymnasium.make( - "catanatron/Catanatron-v0", - config={ - "enemies": [ValueFunctionPlayer(Color.RED)], - "reward_function": reward_fn, - }, + print(f"Creating {N_ENVS} parallel environments...") + + env = make_vec_env( + make_catan_env, + n_envs=N_ENVS, + seed=SEED, + vec_env_cls=SubprocVecEnv, # Use subprocesses for CPU-heavy environments ) - env = ActionMasker(env, mask_fn) - env.reset(seed=SEED) # Load or create model if args.resume: @@ -158,6 +176,7 @@ def main(): # Train print(f"\nTraining for {args.timesteps:,} timesteps") + print(f"With {N_ENVS} parallel environments (~{N_ENVS}x speedup)") print(f"TensorBoard: tensorboard --logdir {TENSORBOARD_DIR}\n") model.learn( @@ -172,6 +191,9 @@ def main(): model.save(final_path) print(f"\nDone! Final model: {final_path}") + # Clean up + env.close() + def mask_fn(env) -> np.ndarray: """Create action mask for valid actions.""" From c21d68a41b75d047dcf8848ffbd97a0706789a54 Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Fri, 19 Dec 2025 21:28:13 -0400 Subject: [PATCH 11/19] Fix Action Space Bug --- catanatron/catanatron/cli/play.py | 2 + catanatron/catanatron/gym/accumulators.py | 15 ++++--- .../catanatron/gym/envs/action_space.py | 44 ++++++++++++------- .../catanatron/gym/envs/catanatron_env.py | 18 +++++--- catanatron/catanatron/models/map.py | 9 ---- .../machine_learning/players/reinforcement.py | 4 +- 6 files changed, 53 insertions(+), 39 deletions(-) diff --git a/catanatron/catanatron/cli/play.py b/catanatron/catanatron/cli/play.py index 00ebd5fc..ef8ddfd3 100644 --- a/catanatron/catanatron/cli/play.py +++ b/catanatron/catanatron/cli/play.py @@ -274,6 +274,7 @@ def play_batch( accumulators.append( CsvDataAccumulator( + (p.color for p in players), game_config.map_type, output_options.output, output_options.include_board_tensor, @@ -285,6 +286,7 @@ def play_batch( accumulators.append( ParquetDataAccumulator( + (p.color for p in players), game_config.map_type, output_options.output, output_options.include_board_tensor, diff --git a/catanatron/catanatron/gym/accumulators.py b/catanatron/catanatron/gym/accumulators.py index c6b23ab0..a7116cff 100644 --- a/catanatron/catanatron/gym/accumulators.py +++ b/catanatron/catanatron/gym/accumulators.py @@ -1,11 +1,12 @@ import os import time from collections import defaultdict -from typing import Literal +from typing import Tuple, Literal import numpy as np import pandas as pd +from catanatron import Action, Color, Game from catanatron.features import create_sample from catanatron.game import GameAccumulator from catanatron.gym.board_tensor_features import create_board_tensor @@ -17,14 +18,13 @@ populate_matrices, simple_total_return, ) -from catanatron.models.actions import Action -from catanatron.game import Game from catanatron.utils import format_secs class ReinforcementLearningAccumulator(GameAccumulator): def __init__( self, + player_colors: Tuple[Color], map_type: Literal["BASE", "TOURNAMENT", "MINI"] = "BASE", include_board_tensor=True, total_return_fns={ @@ -33,6 +33,7 @@ def __init__( "VICTORY_POINTS_RETURN": get_victory_points_total_return, }, ): + self.player_colors = player_colors self.map_type = map_type self.include_board_tensor = include_board_tensor # TODO: Generalize to "rewards_fn" that can yield intermediary rewards @@ -58,7 +59,7 @@ def step(self, game_before_action: Game, action: Action): self.data["samples"].append(create_sample(game_before_action, action.color)) self.data["actions"].append( [ - to_action_space(action, self.map_type), + to_action_space(action, self.player_colors, self.map_type), to_action_type_space(action.action_type), ] ) @@ -140,11 +141,12 @@ def after(self, game): class CsvDataAccumulator(ReinforcementLearningAccumulator): def __init__( self, + player_colors: Tuple[Color], map_type: Literal["BASE", "TOURNAMENT", "MINI"], output, include_board_tensor=True, ): - super().__init__(map_type, include_board_tensor) + super().__init__(player_colors, map_type, include_board_tensor) self.output = output def after(self, game): @@ -179,11 +181,12 @@ def after(self, game): class ParquetDataAccumulator(ReinforcementLearningAccumulator): def __init__( self, + player_colors: Tuple[Color], map_type: Literal["BASE", "TOURNAMENT", "MINI"], output, include_board_tensor=True, ): - super().__init__(map_type, include_board_tensor) + super().__init__(player_colors, map_type, include_board_tensor) self.output = output def after(self, game): diff --git a/catanatron/catanatron/gym/envs/action_space.py b/catanatron/catanatron/gym/envs/action_space.py index c9b2ab53..e9cc8739 100644 --- a/catanatron/catanatron/gym/envs/action_space.py +++ b/catanatron/catanatron/gym/envs/action_space.py @@ -1,23 +1,28 @@ -from typing import Literal from functools import lru_cache +from typing import Tuple, Literal from catanatron.models.actions import Action -from catanatron.models.map import get_map_template, NUM_NODES, LandTile -from catanatron.models.enums import RESOURCES, ActionType from catanatron.models.board import get_edges +from catanatron.models.enums import RESOURCES, ActionType from catanatron.models.player import Color +from catanatron.models.map import build_map @lru_cache(maxsize=None) -def get_action_array(map_type: Literal["BASE", "TOURNAMENT", "MINI"]): - map_template = get_map_template(map_type) - tile_coordinates = [x for x, y in map_template.topology.items() if y == LandTile] +def get_action_array( + player_colors: Tuple[Color], map_type: Literal["BASE", "TOURNAMENT", "MINI"] +): + catan_map = build_map(map_type) + num_nodes = len(catan_map.land_nodes) actions_array = [ (ActionType.ROLL, None), (ActionType.DISCARD, None), - *[(ActionType.BUILD_ROAD, tuple(sorted(edge))) for edge in get_edges()], - *[(ActionType.BUILD_SETTLEMENT, node_id) for node_id in range(NUM_NODES)], - *[(ActionType.BUILD_CITY, node_id) for node_id in range(NUM_NODES)], + *[ + (ActionType.BUILD_ROAD, tuple(sorted(edge))) + for edge in get_edges(catan_map.land_nodes) + ], + *[(ActionType.BUILD_SETTLEMENT, node_id) for node_id in range(num_nodes)], + *[(ActionType.BUILD_CITY, node_id) for node_id in range(num_nodes)], (ActionType.BUY_DEVELOPMENT_CARD, None), (ActionType.PLAY_KNIGHT_CARD, None), *[ @@ -30,9 +35,9 @@ def get_action_array(map_type: Literal["BASE", "TOURNAMENT", "MINI"]): *[(ActionType.PLAY_MONOPOLY, r) for r in RESOURCES], # Move Robber actions include to every tile and from each opponent *[ - (ActionType.MOVE_ROBBER, (tile, victim_color)) - for tile in tile_coordinates - for victim_color in [None] + [color for color in Color] + (ActionType.MOVE_ROBBER, (coordinates, victim_color)) + for coordinates in catan_map.land_tiles.keys() + for victim_color in [None] + list(player_colors) ], # 4:1 with bank *[ @@ -67,19 +72,26 @@ def to_action_type_space(action_type: ActionType) -> int: return ACTION_TYPES.index(action_type) -def to_action_space(action: Action, map_type: Literal["BASE", "TOURNAMENT", "MINI"]): +def to_action_space( + action: Action, + player_colors: Tuple[Color], + map_type: Literal["BASE", "TOURNAMENT", "MINI"], +): """maps action to space_action equivalent integer""" - actions_array = get_action_array(map_type) + actions_array = get_action_array(player_colors, map_type) return actions_array.index((action.action_type, action.value)) def from_action_space( - action_int, playable_actions, map_type: Literal["BASE", "TOURNAMENT", "MINI"] + action_int, + playable_actions, + player_colors: Tuple[Color], + map_type: Literal["BASE", "TOURNAMENT", "MINI"], ): """maps action_int to catantron.models.actions.Action""" # Get "catan_action" based on space action. # i.e. Take first action in playable that matches actions_array blueprint - actions_array = get_action_array(map_type) + actions_array = get_action_array(player_colors, map_type) (action_type, value) = actions_array[action_int] catan_action = None for action in playable_actions: diff --git a/catanatron/catanatron/gym/envs/catanatron_env.py b/catanatron/catanatron/gym/envs/catanatron_env.py index bb1bcea9..d9ff1de4 100644 --- a/catanatron/catanatron/gym/envs/catanatron_env.py +++ b/catanatron/catanatron/gym/envs/catanatron_env.py @@ -54,11 +54,14 @@ def __init__(self, config=None): self.reward_function = self.config.get("reward_function", simple_reward) self.map_type = self.config.get("map_type", "BASE") self.vps_to_win = self.config.get("vps_to_win", 10) - self.enemies = self.config.get("enemies", [RandomPlayer(Color.RED)]) + self.representation = self.config.get("representation", "vector") + assert self.representation in ["mixed", "vector"] + self.enemies = self.config.get("enemies", [RandomPlayer(Color.RED)]) + self.player_colors = tuple([Color.BLUE] + [p.color for p in self.enemies]) assert all(p.color != Color.BLUE for p in self.enemies) - assert self.representation in ["mixed", "vector"] + self.p0 = Player(Color.BLUE) self.players = [self.p0] + self.enemies # type: ignore self.representation = "mixed" if self.representation == "mixed" else "vector" @@ -67,7 +70,7 @@ def __init__(self, config=None): self.max_invalid_actions = 10 # Build action space depending on map type - self.action_array = get_action_array(self.map_type) + self.action_array = get_action_array(self.player_colors, self.map_type) self.action_space_size = len(self.action_array) self.action_space = spaces.Discrete(self.action_space_size) @@ -103,7 +106,10 @@ def get_valid_actions(self): Returns: List[int]: valid actions """ - return [to_action_space(a, self.map_type) for a in self.game.playable_actions] + return [ + to_action_space(a, self.player_colors, self.map_type) + for a in self.game.playable_actions + ] def action_masks(self) -> list[bool]: """ @@ -119,7 +125,7 @@ def action_masks(self) -> list[bool]: def step(self, action): try: catan_action = from_action_space( - action, self.game.playable_actions, self.map_type + action, self.game.playable_actions, self.player_colors, self.map_type ) except Exception: self.invalid_actions_count += 1 @@ -224,7 +230,7 @@ def _advance_until_p0_decision(self): * - Integer - Catanatron Action """ -for i, v in enumerate(get_action_array("BASE")): +for i, v in enumerate(get_action_array((Color.BLUE, Color.RED), "BASE")): CatanatronEnv.__doc__ += f" * - {i}\n - {v}\n" CatanatronEnv.__doc__ += """ diff --git a/catanatron/catanatron/models/map.py b/catanatron/catanatron/models/map.py index 6835ce07..6d8c9a3f 100644 --- a/catanatron/catanatron/models/map.py +++ b/catanatron/catanatron/models/map.py @@ -192,15 +192,6 @@ class MapTemplate: ) -def get_map_template(map_type: Literal["BASE", "TOURNAMENT", "MINI"]): - if map_type == "TOURNAMENT" or map_type == "BASE": - return BASE_MAP_TEMPLATE - elif map_type == "MINI": - return MINI_MAP_TEMPLATE - else: - raise ValueError(f"Invalid map_type: {map_type}") - - class CatanMap: """Represents a randomly initialized map.""" diff --git a/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py b/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py index b92e9978..5b8d9fde 100644 --- a/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py +++ b/catanatron_experimental/catanatron_experimental/machine_learning/players/reinforcement.py @@ -4,7 +4,7 @@ import tensorflow as tf from tensorflow import keras -from catanatron.models.player import Player +from catanatron import Color, Player from catanatron.features import ( create_sample, create_sample_vector, @@ -19,7 +19,7 @@ ) -ACTIONS_ARRAY = get_action_array("BASE") +ACTIONS_ARRAY = get_action_array((Color.BLUE, Color.RED), "BASE") ACTION_SPACE_SIZE = len(ACTIONS_ARRAY) # from catanatron_experimental.rep_b_model import build_model From bda95aa6ff01a3a38cdf91e1cb694ac63566dd69 Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Fri, 19 Dec 2025 21:44:31 -0400 Subject: [PATCH 12/19] Bug Fixes and Tests --- catanatron/catanatron/cli/play.py | 4 +-- tests/test_gym.py | 44 +++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/catanatron/catanatron/cli/play.py b/catanatron/catanatron/cli/play.py index ef8ddfd3..7a1d8307 100644 --- a/catanatron/catanatron/cli/play.py +++ b/catanatron/catanatron/cli/play.py @@ -274,7 +274,7 @@ def play_batch( accumulators.append( CsvDataAccumulator( - (p.color for p in players), + tuple(p.color for p in players), game_config.map_type, output_options.output, output_options.include_board_tensor, @@ -286,7 +286,7 @@ def play_batch( accumulators.append( ParquetDataAccumulator( - (p.color for p in players), + tuple(p.color for p in players), game_config.map_type, output_options.output, output_options.include_board_tensor, diff --git a/tests/test_gym.py b/tests/test_gym.py index e1e235ff..f998c598 100644 --- a/tests/test_gym.py +++ b/tests/test_gym.py @@ -6,8 +6,10 @@ from catanatron.features import get_feature_ordering from catanatron.models.player import Color, RandomPlayer +from catanatron.models.enums import ActionType from catanatron.players.value import ValueFunctionPlayer from catanatron.gym.envs.catanatron_env import CatanatronEnv +from catanatron.gym.envs.action_space import get_action_array features = get_feature_ordering(2) @@ -139,3 +141,45 @@ def test_mixed_rep(): observation, info = env.reset() assert "board" in observation assert "numeric" in observation + + +def test_move_robber_action_in_base_action_array(): + """Test that a specific MOVE_ROBBER action is in the BASE action array for 2 players.""" + player_colors = (Color.BLUE, Color.RED) + action_array = get_action_array(player_colors, "BASE") + target_action = (ActionType.MOVE_ROBBER, ((-1, 0, 1), Color.BLUE)) + assert target_action in action_array, ( + f"Action {target_action} not found in BASE action array for 2 players" + ) + + target_action = (ActionType.MOVE_ROBBER, ((-1, 0, 1), None)) + assert target_action in action_array, ( + f"Action {target_action} not found in BASE action array for 2 players" + ) + + +def test_there_are_54_build_nodes_in_base(): + player_colors = (Color.BLUE, Color.RED) + action_array = get_action_array(player_colors, "BASE") + num_build_nodes = len( + [action for action in action_array if action[0] == ActionType.BUILD_SETTLEMENT] + ) + assert num_build_nodes == 54 + + +def test_there_are_less_build_nodes_in_mini(): + player_colors = (Color.BLUE, Color.RED) + action_array = get_action_array(player_colors, "MINI") + num_build_nodes = len( + [action for action in action_array if action[0] == ActionType.BUILD_SETTLEMENT] + ) + assert num_build_nodes == 24 + + +def test_outside_tiles_not_in_mini(): + player_colors = (Color.BLUE, Color.RED) + action_array = get_action_array(player_colors, "MINI") + target_action = (ActionType.MOVE_ROBBER, ((0, 2, -2), Color.BLUE)) + assert target_action not in action_array, ( + f"Action {target_action} found in MINI action array for 2 players" + ) From 64e0e56b618a5bf798f196477a91c6f85ae14102 Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Sat, 20 Dec 2025 08:34:05 -0400 Subject: [PATCH 13/19] Action space refactor and tests --- .../catanatron/gym/envs/action_space.py | 12 +---- .../catanatron/gym/envs/catanatron_env.py | 5 +- tests/test_gym.py | 52 ++++++++++++++++++- 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/catanatron/catanatron/gym/envs/action_space.py b/catanatron/catanatron/gym/envs/action_space.py index e9cc8739..e617a087 100644 --- a/catanatron/catanatron/gym/envs/action_space.py +++ b/catanatron/catanatron/gym/envs/action_space.py @@ -84,19 +84,11 @@ def to_action_space( def from_action_space( action_int, - playable_actions, + color: Color, player_colors: Tuple[Color], map_type: Literal["BASE", "TOURNAMENT", "MINI"], ): """maps action_int to catantron.models.actions.Action""" - # Get "catan_action" based on space action. - # i.e. Take first action in playable that matches actions_array blueprint actions_array = get_action_array(player_colors, map_type) (action_type, value) = actions_array[action_int] - catan_action = None - for action in playable_actions: - if action.action_type == action_type and action.value == value: - catan_action = action - break # return the first one - assert catan_action is not None - return catan_action + return Action(color, action_type, value) diff --git a/catanatron/catanatron/gym/envs/catanatron_env.py b/catanatron/catanatron/gym/envs/catanatron_env.py index d9ff1de4..7c6e17d0 100644 --- a/catanatron/catanatron/gym/envs/catanatron_env.py +++ b/catanatron/catanatron/gym/envs/catanatron_env.py @@ -125,9 +125,10 @@ def action_masks(self) -> list[bool]: def step(self, action): try: catan_action = from_action_space( - action, self.game.playable_actions, self.player_colors, self.map_type + action, self.p0.color, self.player_colors, self.map_type ) - except Exception: + assert catan_action in self.game.playable_actions + except AssertionError: self.invalid_actions_count += 1 observation = self._get_observation() diff --git a/tests/test_gym.py b/tests/test_gym.py index f998c598..f2790f95 100644 --- a/tests/test_gym.py +++ b/tests/test_gym.py @@ -6,10 +6,14 @@ from catanatron.features import get_feature_ordering from catanatron.models.player import Color, RandomPlayer -from catanatron.models.enums import ActionType +from catanatron.models.enums import Action, ActionType, WHEAT, SHEEP, ORE from catanatron.players.value import ValueFunctionPlayer from catanatron.gym.envs.catanatron_env import CatanatronEnv -from catanatron.gym.envs.action_space import get_action_array +from catanatron.gym.envs.action_space import ( + get_action_array, + to_action_space, + from_action_space, +) features = get_feature_ordering(2) @@ -183,3 +187,47 @@ def test_outside_tiles_not_in_mini(): assert target_action not in action_array, ( f"Action {target_action} found in MINI action array for 2 players" ) + + +def test_action_space_conversion_roundtrip(): + """Test converting actions to action space integers and back.""" + player_colors = (Color.BLUE, Color.RED) + map_type = "BASE" + + # Create various test actions + # Note: For PLAY_YEAR_OF_PLENTY with 2 resources, they must be in RESOURCES order + # RESOURCES = ['WOOD', 'BRICK', 'SHEEP', 'WHEAT', 'ORE'] + test_actions = [ + Action(Color.BLUE, ActionType.ROLL, None), + Action(Color.BLUE, ActionType.DISCARD, None), + Action(Color.BLUE, ActionType.BUILD_SETTLEMENT, 10), + Action(Color.BLUE, ActionType.BUILD_CITY, 5), + Action(Color.BLUE, ActionType.BUILD_ROAD, (0, 1)), + Action(Color.BLUE, ActionType.BUY_DEVELOPMENT_CARD, None), + Action(Color.BLUE, ActionType.PLAY_KNIGHT_CARD, None), + Action(Color.BLUE, ActionType.PLAY_YEAR_OF_PLENTY, (SHEEP, WHEAT)), + Action(Color.BLUE, ActionType.PLAY_YEAR_OF_PLENTY, (WHEAT, ORE)), + Action(Color.BLUE, ActionType.PLAY_YEAR_OF_PLENTY, (ORE,)), + Action(Color.BLUE, ActionType.PLAY_MONOPOLY, WHEAT), + Action(Color.BLUE, ActionType.PLAY_ROAD_BUILDING, None), + Action(Color.BLUE, ActionType.MOVE_ROBBER, ((-1, 0, 1), Color.RED)), + Action(Color.BLUE, ActionType.MOVE_ROBBER, ((0, -1, 1), None)), + Action(Color.BLUE, ActionType.MARITIME_TRADE, (WHEAT, WHEAT, WHEAT, WHEAT, ORE)), + Action(Color.BLUE, ActionType.MARITIME_TRADE, (SHEEP, SHEEP, SHEEP, None, WHEAT)), + Action(Color.BLUE, ActionType.MARITIME_TRADE, (ORE, ORE, None, None, WHEAT)), + Action(Color.BLUE, ActionType.END_TURN, None), + ] + + for action in test_actions: + # Convert to action space integer + action_int = to_action_space(action, player_colors, map_type) + + # Convert back from action space + recovered_action = from_action_space( + action_int, action.color, player_colors, map_type + ) + + # Assert they are the same + assert recovered_action == action, ( + f"Action conversion failed: {action} -> {action_int} -> {recovered_action}" + ) From 1d444e1a946e4c1581bb545f2c5786f33f0533f1 Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Sat, 20 Dec 2025 08:57:39 -0400 Subject: [PATCH 14/19] LR Schedule --- examples/ppo/train.py | 94 ++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 38 deletions(-) diff --git a/examples/ppo/train.py b/examples/ppo/train.py index 78d9a094..7346d036 100644 --- a/examples/ppo/train.py +++ b/examples/ppo/train.py @@ -4,18 +4,21 @@ Features: - Vectorized environments (8 parallel games for ~8x speedup) - Shaped reward function (incremental rewards for progress) +- Linear learning rate schedule (decreases over time) - GPU support (automatic detection) - Checkpoint saving/loading for resumable training - TensorBoard integration for monitoring Configure by editing constants: - N_ENVS = 8 # Parallel environments - NUM_LAYERS = 3 # Number of hidden layers - NEURONS_PER_LAYER = 256 # Neurons in each layer - N_STEPS = 2048 # PPO rollout steps - BATCH_SIZE = 256 # PPO batch size - ENT_COEF = 0.01 # Exploration (higher = more exploration) - USE_SHAPED_REWARD = True # Incremental vs sparse rewards + N_ENVS = 8 # Parallel environments + NUM_LAYERS = 3 # Number of hidden layers + NEURONS_PER_LAYER = 256 # Neurons in each layer + N_STEPS = 2048 # PPO rollout steps + BATCH_SIZE = 256 # PPO batch size + INITIAL_LR = 0.0003 # Initial learning rate + FINAL_LR = 0.00001 # Final learning rate + ENT_COEF = 0.05 # Entropy coefficient for exploration + USE_SHAPED_REWARD = True # Incremental vs sparse rewards Usage: # Start new training: @@ -63,30 +66,18 @@ USE_SHAPED_REWARD = True # PPO parameters -ENT_COEF = 0.01 # Entropy coefficient for exploration (default: 0.0, try 0.01-0.1 for more exploration) -N_ENVS = 1 # Number of parallel environments +N_ENVS = 8 # Number of parallel environments N_STEPS = 2048 # Number of steps to collect before update (default: 2048) BATCH_SIZE = 256 # Batch size for training (default: 64) assert (N_ENVS * N_STEPS) % BATCH_SIZE == 0, "BATCH_SIZE must divide N_ENVS * N_STEPS" - - -def make_catan_env(): - """Factory function to create a Catan environment for vectorization.""" - # Create fresh reward function instance for each environment - reward_fn = ShapedRewardFunction() if USE_SHAPED_REWARD else simple_reward - - env = gymnasium.make( - "catanatron/Catanatron-v0", - config={ - "enemies": [ValueFunctionPlayer(Color.RED)], - "reward_function": reward_fn, - }, - ) - env = ActionMasker(env, mask_fn) - return env +# Learning rate schedule +INITIAL_LR = 0.0003 # Initial learning rate +FINAL_LR = 0.00001 # Final learning rate +ENT_COEF = 0.05 # Entropy coefficient for exploration def main(): + # Parse arguments parser = argparse.ArgumentParser() parser.add_argument("--resume", action="store_true", help="Resume from checkpoint") parser.add_argument( @@ -113,16 +104,11 @@ def main(): ) print(f"Using device: {device}") - # Create directories - os.makedirs(CHECKPOINT_DIR, exist_ok=True) - os.makedirs(TENSORBOARD_DIR, exist_ok=True) - # Create vectorized environments print( f"Using reward function: {'shaped (incremental)' if USE_SHAPED_REWARD else 'simple (sparse)'}" ) print(f"Creating {N_ENVS} parallel environments...") - env = make_vec_env( make_catan_env, n_envs=N_ENVS, @@ -131,6 +117,9 @@ def main(): ) # Load or create model + # Create directories + os.makedirs(CHECKPOINT_DIR, exist_ok=True) + os.makedirs(TENSORBOARD_DIR, exist_ok=True) if args.resume: checkpoint = get_latest_checkpoint() if checkpoint: @@ -150,13 +139,18 @@ def main(): net_arch = [NEURONS_PER_LAYER] * NUM_LAYERS policy_kwargs = dict(net_arch=net_arch) + # Create learning rate schedule + lr_schedule = linear_schedule(INITIAL_LR, FINAL_LR) + print(f"Creating new model with architecture: {net_arch}") print( f"PPO config: n_steps={N_STEPS}, batch_size={BATCH_SIZE}, ent_coef={ENT_COEF}" ) + print(f"Learning rate schedule: {INITIAL_LR:.2e} → {FINAL_LR:.2e}") model = MaskablePPO( MaskableActorCriticPolicy, env, + learning_rate=lr_schedule, n_steps=N_STEPS, batch_size=BATCH_SIZE, ent_coef=ENT_COEF, @@ -195,14 +189,6 @@ def main(): env.close() -def mask_fn(env) -> np.ndarray: - """Create action mask for valid actions.""" - valid_actions = env.unwrapped.get_valid_actions() - mask = np.zeros(env.action_space.n, dtype=np.float32) - mask[valid_actions] = 1 - return np.array([bool(i) for i in mask]) - - def get_latest_checkpoint(): """Find the most recent checkpoint.""" checkpoint_path = Path(CHECKPOINT_DIR) @@ -218,5 +204,37 @@ def get_latest_checkpoint(): return str(latest) +def linear_schedule(initial_value, final_value): + def schedule(progress_remaining): + # progress_remaining goes from 1.0 (start) to 0.0 (end) + return final_value + progress_remaining * (initial_value - final_value) + + return schedule + + +def make_catan_env(): + """Factory function to create a Catan environment for vectorization.""" + # Create fresh reward function instance for each environment + reward_fn = ShapedRewardFunction() if USE_SHAPED_REWARD else simple_reward + + env = gymnasium.make( + "catanatron/Catanatron-v0", + config={ + "enemies": [ValueFunctionPlayer(Color.RED)], + "reward_function": reward_fn, + }, + ) + env = ActionMasker(env, mask_fn) + return env + + +def mask_fn(env) -> np.ndarray: + """Create action mask for valid actions.""" + valid_actions = env.unwrapped.get_valid_actions() + mask = np.zeros(env.action_space.n, dtype=np.float32) + mask[valid_actions] = 1 + return np.array([bool(i) for i in mask]) + + if __name__ == "__main__": main() From a0669f35fd5f67a00cfce859394e72f80c1dd74d Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Sun, 21 Dec 2025 03:47:07 -0400 Subject: [PATCH 15/19] VecNormalize --- examples/ppo/train.py | 56 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/examples/ppo/train.py b/examples/ppo/train.py index 7346d036..4e33dd6f 100644 --- a/examples/ppo/train.py +++ b/examples/ppo/train.py @@ -3,6 +3,7 @@ Features: - Vectorized environments (8 parallel games for ~8x speedup) +- VecNormalize (observation and reward normalization) - Shaped reward function (incremental rewards for progress) - Linear learning rate schedule (decreases over time) - GPU support (automatic detection) @@ -47,7 +48,7 @@ from sb3_contrib.ppo_mask import MaskablePPO from stable_baselines3.common.callbacks import CheckpointCallback from stable_baselines3.common.env_util import make_vec_env -from stable_baselines3.common.vec_env import SubprocVecEnv +from stable_baselines3.common.vec_env import SubprocVecEnv, VecNormalize import catanatron.gym from catanatron import Color @@ -71,8 +72,8 @@ BATCH_SIZE = 256 # Batch size for training (default: 64) assert (N_ENVS * N_STEPS) % BATCH_SIZE == 0, "BATCH_SIZE must divide N_ENVS * N_STEPS" # Learning rate schedule -INITIAL_LR = 0.0003 # Initial learning rate -FINAL_LR = 0.00001 # Final learning rate +INITIAL_LR = 0.03 # Initial learning rate +FINAL_LR = 0.0001 # Final learning rate ENT_COEF = 0.05 # Entropy coefficient for exploration @@ -116,6 +117,16 @@ def main(): vec_env_cls=SubprocVecEnv, # Use subprocesses for CPU-heavy environments ) + # Wrap with VecNormalize for observation and reward normalization + print("Wrapping environments with VecNormalize (obs + reward normalization)") + env = VecNormalize( + env, + norm_obs=True, # Normalize observations + norm_reward=True, # Normalize rewards + clip_obs=10.0, # Clip observations to [-10, 10] after normalization + clip_reward=10.0, # Clip rewards to [-10, 10] after normalization + ) + # Load or create model # Create directories os.makedirs(CHECKPOINT_DIR, exist_ok=True) @@ -124,6 +135,13 @@ def main(): checkpoint = get_latest_checkpoint() if checkpoint: print(f"Loading checkpoint: {checkpoint}") + + # Load VecNormalize stats if available + vec_normalize_path = checkpoint.replace(".zip", "_vecnormalize.pkl") + if os.path.exists(vec_normalize_path): + print(f"Loading VecNormalize stats from: {vec_normalize_path}") + env = VecNormalize.load(vec_normalize_path, env) + model = MaskablePPO.load( checkpoint, env=env, @@ -161,8 +179,8 @@ def main(): device=device, ) - # Setup checkpoint callback - checkpoint_callback = CheckpointCallback( + # Setup checkpoint callback (also saves VecNormalize stats) + checkpoint_callback = VecNormalizeCheckpointCallback( save_freq=CHECKPOINT_FREQ, save_path=CHECKPOINT_DIR, name_prefix="rl_model", @@ -180,10 +198,15 @@ def main(): tb_log_name="MaskablePPO", ) - # Save final model + # Save final model and VecNormalize stats final_path = os.path.join(CHECKPOINT_DIR, "final_model.zip") model.save(final_path) + + vec_normalize_path = os.path.join(CHECKPOINT_DIR, "final_model_vecnormalize.pkl") + env.save(vec_normalize_path) + print(f"\nDone! Final model: {final_path}") + print(f"VecNormalize stats: {vec_normalize_path}") # Clean up env.close() @@ -212,6 +235,27 @@ def schedule(progress_remaining): return schedule +class VecNormalizeCheckpointCallback(CheckpointCallback): + """Custom checkpoint callback that also saves VecNormalize statistics.""" + + def _on_step(self) -> bool: + # Save model checkpoint (parent class behavior) + result = super()._on_step() + + # Also save VecNormalize stats if the model was just saved + if result and isinstance(self.model.get_env(), VecNormalize): + # Get the path of the most recently saved model + checkpoint_path = os.path.join( + self.save_path, f"{self.name_prefix}_{self.num_timesteps}_steps.zip" + ) + vec_normalize_path = checkpoint_path.replace(".zip", "_vecnormalize.pkl") + + # Save VecNormalize statistics + self.model.get_env().save(vec_normalize_path) + + return result + + def make_catan_env(): """Factory function to create a Catan environment for vectorization.""" # Create fresh reward function instance for each environment From 5b9aa5cd6cd103c9a9fb5557cc39230b21a1abe4 Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Sun, 21 Dec 2025 14:29:19 -0400 Subject: [PATCH 16/19] Experiment Name --- examples/ppo/train.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/examples/ppo/train.py b/examples/ppo/train.py index 4e33dd6f..00a4e7ca 100644 --- a/examples/ppo/train.py +++ b/examples/ppo/train.py @@ -67,14 +67,14 @@ USE_SHAPED_REWARD = True # PPO parameters -N_ENVS = 8 # Number of parallel environments +N_ENVS = 32 # Number of parallel environments N_STEPS = 2048 # Number of steps to collect before update (default: 2048) BATCH_SIZE = 256 # Batch size for training (default: 64) assert (N_ENVS * N_STEPS) % BATCH_SIZE == 0, "BATCH_SIZE must divide N_ENVS * N_STEPS" # Learning rate schedule -INITIAL_LR = 0.03 # Initial learning rate +INITIAL_LR = 0.01 # Initial learning rate FINAL_LR = 0.0001 # Final learning rate -ENT_COEF = 0.05 # Entropy coefficient for exploration +ENT_COEF = 0.01 # Entropy coefficient for exploration def main(): @@ -122,7 +122,7 @@ def main(): env = VecNormalize( env, norm_obs=True, # Normalize observations - norm_reward=True, # Normalize rewards + norm_reward=False, # Normalize rewards clip_obs=10.0, # Clip observations to [-10, 10] after normalization clip_reward=10.0, # Clip rewards to [-10, 10] after normalization ) @@ -191,11 +191,19 @@ def main(): print(f"With {N_ENVS} parallel environments (~{N_ENVS}x speedup)") print(f"TensorBoard: tensorboard --logdir {TENSORBOARD_DIR}\n") + # Create experiment name with parameters + reward_type = "shaped" if USE_SHAPED_REWARD else "simple" + experiment_name = ( + f"MaskablePPO_envs{N_ENVS}_steps{N_STEPS}_batch{BATCH_SIZE}_" + f"lr{INITIAL_LR}-{FINAL_LR}_ent{ENT_COEF}_" + f"layers{NUM_LAYERS}x{NEURONS_PER_LAYER}_{reward_type}" + ) + model.learn( total_timesteps=args.timesteps, callback=checkpoint_callback, reset_num_timesteps=not args.resume, - tb_log_name="MaskablePPO", + tb_log_name=experiment_name, ) # Save final model and VecNormalize stats From 6808881c818e229c02fd6bb90a2886a1c9053e02 Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Sun, 21 Dec 2025 14:38:22 -0400 Subject: [PATCH 17/19] Config Object and go MINI 6 --- examples/ppo/train.py | 114 +++++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 51 deletions(-) diff --git a/examples/ppo/train.py b/examples/ppo/train.py index 00a4e7ca..88a3327a 100644 --- a/examples/ppo/train.py +++ b/examples/ppo/train.py @@ -2,24 +2,14 @@ Restartable Stable Baselines3 training example with TensorBoard logging. Features: -- Vectorized environments (8 parallel games for ~8x speedup) +- Vectorized environments (parallel games for speedup) - VecNormalize (observation and reward normalization) - Shaped reward function (incremental rewards for progress) - Linear learning rate schedule (decreases over time) - GPU support (automatic detection) - Checkpoint saving/loading for resumable training - TensorBoard integration for monitoring - -Configure by editing constants: - N_ENVS = 8 # Parallel environments - NUM_LAYERS = 3 # Number of hidden layers - NEURONS_PER_LAYER = 256 # Neurons in each layer - N_STEPS = 2048 # PPO rollout steps - BATCH_SIZE = 256 # PPO batch size - INITIAL_LR = 0.0003 # Initial learning rate - FINAL_LR = 0.00001 # Final learning rate - ENT_COEF = 0.05 # Entropy coefficient for exploration - USE_SHAPED_REWARD = True # Incremental vs sparse rewards +- Wandb-ready configuration object Usage: # Start new training: @@ -33,6 +23,10 @@ # View TensorBoard: tensorboard --logdir ./tensorboard_logs + + # To use with wandb (add to main()): + import wandb + wandb.init(project="catan-ppo", config=config) """ import random @@ -57,24 +51,35 @@ from shaped_reward import ShapedRewardFunction -# Configuration -SEED = 42 +# Configuration object (compatible with wandb) +config = { + # Environment parameters + "map_type": "MINI", # Map type for Catan (BASE, MINI, etc.) + "vps_to_win": 6, # Victory points needed to win + "use_shaped_reward": True, # Use shaped vs simple reward function + # PPO hyperparameters + "n_envs": 32, # Number of parallel environments + "n_steps": 2048, # Number of steps to collect before update + "batch_size": 256, # Batch size for training + "initial_lr": 0.01, # Initial learning rate + "final_lr": 0.001, # Final learning rate + "ent_coef": 0.01, # Entropy coefficient for exploration + # Network architecture + "num_layers": 3, # Number of hidden layers + "neurons_per_layer": 256, # Neurons in each layer + # Training parameters + "seed": 42, # Random seed + "checkpoint_freq": 10_000, # Save checkpoint every N steps +} + +# Validate configuration +assert (config["n_envs"] * config["n_steps"]) % config["batch_size"] == 0, ( + "BATCH_SIZE must divide N_ENVS * N_STEPS" +) + +# Directories CHECKPOINT_DIR = os.path.join(os.path.dirname(__file__), "checkpoints") TENSORBOARD_DIR = os.path.join(os.path.dirname(__file__), "tensorboard_logs") -CHECKPOINT_FREQ = 10_000 -NUM_LAYERS = 3 -NEURONS_PER_LAYER = 256 -USE_SHAPED_REWARD = True - -# PPO parameters -N_ENVS = 32 # Number of parallel environments -N_STEPS = 2048 # Number of steps to collect before update (default: 2048) -BATCH_SIZE = 256 # Batch size for training (default: 64) -assert (N_ENVS * N_STEPS) % BATCH_SIZE == 0, "BATCH_SIZE must divide N_ENVS * N_STEPS" -# Learning rate schedule -INITIAL_LR = 0.01 # Initial learning rate -FINAL_LR = 0.0001 # Final learning rate -ENT_COEF = 0.01 # Entropy coefficient for exploration def main(): @@ -90,10 +95,10 @@ def main(): args = parser.parse_args() # Set random seeds for reproducibility - random.seed(SEED) - np.random.seed(SEED) - torch.manual_seed(SEED) - torch.cuda.manual_seed_all(SEED) + random.seed(config["seed"]) + np.random.seed(config["seed"]) + torch.manual_seed(config["seed"]) + torch.cuda.manual_seed_all(config["seed"]) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False @@ -107,13 +112,14 @@ def main(): # Create vectorized environments print( - f"Using reward function: {'shaped (incremental)' if USE_SHAPED_REWARD else 'simple (sparse)'}" + f"Using reward function: {'shaped (incremental)' if config['use_shaped_reward'] else 'simple (sparse)'}" ) - print(f"Creating {N_ENVS} parallel environments...") + print(f"Using map type: {config['map_type']}, VPs to win: {config['vps_to_win']}") + print(f"Creating {config['n_envs']} parallel environments...") env = make_vec_env( make_catan_env, - n_envs=N_ENVS, - seed=SEED, + n_envs=config["n_envs"], + seed=config["seed"], vec_env_cls=SubprocVecEnv, # Use subprocesses for CPU-heavy environments ) @@ -154,49 +160,53 @@ def main(): if not args.resume: # Configure network architecture - net_arch = [NEURONS_PER_LAYER] * NUM_LAYERS + net_arch = [config["neurons_per_layer"]] * config["num_layers"] policy_kwargs = dict(net_arch=net_arch) # Create learning rate schedule - lr_schedule = linear_schedule(INITIAL_LR, FINAL_LR) + lr_schedule = linear_schedule(config["initial_lr"], config["final_lr"]) print(f"Creating new model with architecture: {net_arch}") print( - f"PPO config: n_steps={N_STEPS}, batch_size={BATCH_SIZE}, ent_coef={ENT_COEF}" + f"PPO config: n_steps={config['n_steps']}, batch_size={config['batch_size']}, ent_coef={config['ent_coef']}" + ) + print( + f"Learning rate schedule: {config['initial_lr']:.2e} → {config['final_lr']:.2e}" ) - print(f"Learning rate schedule: {INITIAL_LR:.2e} → {FINAL_LR:.2e}") model = MaskablePPO( MaskableActorCriticPolicy, env, learning_rate=lr_schedule, - n_steps=N_STEPS, - batch_size=BATCH_SIZE, - ent_coef=ENT_COEF, + n_steps=config["n_steps"], + batch_size=config["batch_size"], + ent_coef=config["ent_coef"], verbose=1, tensorboard_log=TENSORBOARD_DIR, - seed=SEED, + seed=config["seed"], policy_kwargs=policy_kwargs, device=device, ) # Setup checkpoint callback (also saves VecNormalize stats) checkpoint_callback = VecNormalizeCheckpointCallback( - save_freq=CHECKPOINT_FREQ, + save_freq=config["checkpoint_freq"], save_path=CHECKPOINT_DIR, name_prefix="rl_model", ) # Train print(f"\nTraining for {args.timesteps:,} timesteps") - print(f"With {N_ENVS} parallel environments (~{N_ENVS}x speedup)") + print( + f"With {config['n_envs']} parallel environments (~{config['n_envs']}x speedup)" + ) print(f"TensorBoard: tensorboard --logdir {TENSORBOARD_DIR}\n") # Create experiment name with parameters - reward_type = "shaped" if USE_SHAPED_REWARD else "simple" + reward_type = "shaped" if config["use_shaped_reward"] else "simple" experiment_name = ( - f"MaskablePPO_envs{N_ENVS}_steps{N_STEPS}_batch{BATCH_SIZE}_" - f"lr{INITIAL_LR}-{FINAL_LR}_ent{ENT_COEF}_" - f"layers{NUM_LAYERS}x{NEURONS_PER_LAYER}_{reward_type}" + f"MaskablePPO_{config['map_type']}_vp{config['vps_to_win']}_envs{config['n_envs']}_steps{config['n_steps']}_batch{config['batch_size']}_" + f"lr{config['initial_lr']}-{config['final_lr']}_ent{config['ent_coef']}_" + f"layers{config['num_layers']}x{config['neurons_per_layer']}_{reward_type}" ) model.learn( @@ -267,11 +277,13 @@ def _on_step(self) -> bool: def make_catan_env(): """Factory function to create a Catan environment for vectorization.""" # Create fresh reward function instance for each environment - reward_fn = ShapedRewardFunction() if USE_SHAPED_REWARD else simple_reward + reward_fn = ShapedRewardFunction() if config["use_shaped_reward"] else simple_reward env = gymnasium.make( "catanatron/Catanatron-v0", config={ + "map_type": config["map_type"], + "vps_to_win": config["vps_to_win"], "enemies": [ValueFunctionPlayer(Color.RED)], "reward_function": reward_fn, }, From 541aaed7991d637e3033f545ccc9a00cf79c669e Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Sun, 21 Dec 2025 15:35:42 -0400 Subject: [PATCH 18/19] Add Gamma Param --- examples/ppo/train.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/ppo/train.py b/examples/ppo/train.py index 88a3327a..93014b74 100644 --- a/examples/ppo/train.py +++ b/examples/ppo/train.py @@ -61,6 +61,7 @@ "n_envs": 32, # Number of parallel environments "n_steps": 2048, # Number of steps to collect before update "batch_size": 256, # Batch size for training + "gamma": 0.99, # Discount factor for future rewards "initial_lr": 0.01, # Initial learning rate "final_lr": 0.001, # Final learning rate "ent_coef": 0.01, # Entropy coefficient for exploration @@ -168,7 +169,7 @@ def main(): print(f"Creating new model with architecture: {net_arch}") print( - f"PPO config: n_steps={config['n_steps']}, batch_size={config['batch_size']}, ent_coef={config['ent_coef']}" + f"PPO config: n_steps={config['n_steps']}, batch_size={config['batch_size']}, gamma={config['gamma']}, ent_coef={config['ent_coef']}" ) print( f"Learning rate schedule: {config['initial_lr']:.2e} → {config['final_lr']:.2e}" @@ -179,6 +180,7 @@ def main(): learning_rate=lr_schedule, n_steps=config["n_steps"], batch_size=config["batch_size"], + gamma=config["gamma"], ent_coef=config["ent_coef"], verbose=1, tensorboard_log=TENSORBOARD_DIR, @@ -205,7 +207,7 @@ def main(): reward_type = "shaped" if config["use_shaped_reward"] else "simple" experiment_name = ( f"MaskablePPO_{config['map_type']}_vp{config['vps_to_win']}_envs{config['n_envs']}_steps{config['n_steps']}_batch{config['batch_size']}_" - f"lr{config['initial_lr']}-{config['final_lr']}_ent{config['ent_coef']}_" + f"gamma{config['gamma']}_lr{config['initial_lr']}-{config['final_lr']}_ent{config['ent_coef']}_" f"layers{config['num_layers']}x{config['neurons_per_layer']}_{reward_type}" ) From 769ebeb9ebedc653c61b408c09a824e0c9f00422 Mon Sep 17 00:00:00 2001 From: Bryan Collazo Date: Sun, 21 Dec 2025 21:03:52 -0400 Subject: [PATCH 19/19] n_epochs --- examples/ppo/train.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/ppo/train.py b/examples/ppo/train.py index 93014b74..2e4136a2 100644 --- a/examples/ppo/train.py +++ b/examples/ppo/train.py @@ -59,8 +59,9 @@ "use_shaped_reward": True, # Use shaped vs simple reward function # PPO hyperparameters "n_envs": 32, # Number of parallel environments - "n_steps": 2048, # Number of steps to collect before update - "batch_size": 256, # Batch size for training + "n_steps": 256, # Number of steps to collect before update + "batch_size": 1024, # Batch size for training + "n_epochs": 5, # Number of epochs for PPO update "gamma": 0.99, # Discount factor for future rewards "initial_lr": 0.01, # Initial learning rate "final_lr": 0.001, # Final learning rate @@ -169,7 +170,7 @@ def main(): print(f"Creating new model with architecture: {net_arch}") print( - f"PPO config: n_steps={config['n_steps']}, batch_size={config['batch_size']}, gamma={config['gamma']}, ent_coef={config['ent_coef']}" + f"PPO config: n_steps={config['n_steps']}, batch_size={config['batch_size']}, n_epochs={config['n_epochs']}, gamma={config['gamma']}, ent_coef={config['ent_coef']}" ) print( f"Learning rate schedule: {config['initial_lr']:.2e} → {config['final_lr']:.2e}" @@ -181,6 +182,7 @@ def main(): n_steps=config["n_steps"], batch_size=config["batch_size"], gamma=config["gamma"], + n_epochs=config["n_epochs"], ent_coef=config["ent_coef"], verbose=1, tensorboard_log=TENSORBOARD_DIR, @@ -207,7 +209,7 @@ def main(): reward_type = "shaped" if config["use_shaped_reward"] else "simple" experiment_name = ( f"MaskablePPO_{config['map_type']}_vp{config['vps_to_win']}_envs{config['n_envs']}_steps{config['n_steps']}_batch{config['batch_size']}_" - f"gamma{config['gamma']}_lr{config['initial_lr']}-{config['final_lr']}_ent{config['ent_coef']}_" + f"gamma{config['gamma']}_epochs{config['n_epochs']}_lr{config['initial_lr']}-{config['final_lr']}_ent{config['ent_coef']}_" f"layers{config['num_layers']}x{config['neurons_per_layer']}_{reward_type}" )