From fa2aff717fa2ba3841b2a29d1d516cda1002abb1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 20 Aug 2025 18:39:42 -0400 Subject: [PATCH 001/238] feat(deployment): Migrate package orchestration to Docker Compose (resolves #1177). --- .../clp_package_utils/general.py | 14 +- .../clp_package_utils/scripts/start_clp.py | 1062 +++-------------- tools/deployment/package/docker-compose.yml | 374 ++++++ 3 files changed, 570 insertions(+), 880 deletions(-) create mode 100644 tools/deployment/package/docker-compose.yml diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 1ae8dbb66e..5191caee23 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -150,6 +150,12 @@ def check_dependencies(): ) except subprocess.CalledProcessError: raise EnvironmentError("docker cannot run without superuser privileges (sudo).") + try: + subprocess.run( + ["docker", "compose", "version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True + ) + except subprocess.CalledProcessError: + raise EnvironmentError("docker-compose is not installed") def is_container_running(container_name): @@ -432,10 +438,10 @@ def load_config_file( validate_path_for_container_mount(clp_config.data_directory) validate_path_for_container_mount(clp_config.logs_directory) - # Make data and logs directories node-specific - hostname = socket.gethostname() - clp_config.data_directory /= hostname - clp_config.logs_directory /= hostname + # # Make data and logs directories node-specific + # hostname = socket.gethostname() + # clp_config.data_directory /= hostname + # clp_config.logs_directory /= hostname return clp_config diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 24a45e7dc0..1938211ca0 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -75,6 +75,10 @@ ) logger = logging.getLogger(__file__) +env_dict = {} + +def get_ip_from_hostname(hostname: str) -> str: + return socket.gethostbyname(hostname) def container_exists(container_name): @@ -156,13 +160,9 @@ def wait_for_container_cmd(container_name: str, cmd_to_run: [str], timeout: int) return False -def start_db(instance_id: str, clp_config: CLPConfig, conf_dir: pathlib.Path): +def start_db(clp_config: CLPConfig, conf_dir: pathlib.Path): component_name = DB_COMPONENT_NAME - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return + logger.info(f"Initializing {component_name}...") db_data_dir = clp_config.data_directory / component_name db_logs_dir = clp_config.logs_directory / component_name @@ -174,321 +174,69 @@ def start_db(instance_id: str, clp_config: CLPConfig, conf_dir: pathlib.Path): db_logs_dir.mkdir(exist_ok=True, parents=True) # Start container - mounts = [ - DockerMount( - DockerMountType.BIND, - conf_dir / "mysql" / "conf.d", - pathlib.Path("/") / "etc" / "mysql" / "conf.d", - True, - ), - DockerMount(DockerMountType.BIND, db_data_dir, pathlib.Path("/") / "var" / "lib" / "mysql"), - DockerMount(DockerMountType.BIND, db_logs_dir, pathlib.Path("/") / "var" / "log" / "mysql"), - ] - env_vars = [ - f"MYSQL_ROOT_PASSWORD={clp_config.database.password}", - f"MYSQL_USER={clp_config.database.username}", - f"MYSQL_PASSWORD={clp_config.database.password}", - f"MYSQL_DATABASE={clp_config.database.name}", - ] - # fmt: off - cmd = [ - "docker", "run", - "-d", - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - append_docker_options(cmd, mounts, env_vars) - append_docker_port_settings_for_host_ips( - clp_config.database.host, clp_config.database.port, 3306, cmd - ) + env_dict["CLP_HOST_DB_CONF_DIR"] = str(conf_dir / "mysql" / "conf.d") + env_dict["CLP_HOST_DB_DATA_DIR"] = str(db_data_dir) + env_dict["CLP_HOST_DB_LOGS_DIR"] = str(db_logs_dir) + + # TODO: Consider a Docker Compose overwrite / extend approach for pick the right image. if "mysql" == clp_config.database.type: - cmd.append("mysql:8.0.23") + env_dict["CLP_DB_IMAGE"] = "mysql:8.0.23" elif "mariadb" == clp_config.database.type: - cmd.append("mariadb:10-jammy") - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) + env_dict["CLP_DB_IMAGE"] = "mariadb:10-jammy" - # fmt: off - mysqladmin_cmd = [ - "mysqladmin", "ping", - "--silent", - "-h", "127.0.0.1", - "-u", str(clp_config.database.username), - f"--password={clp_config.database.password}", - ] - # fmt: on + env_dict["CLP_DB_HOST"] = get_ip_from_hostname(clp_config.database.host) + env_dict["CLP_DB_PORT"] = str(clp_config.database.port) + env_dict["CLP_DB_NAME"] = clp_config.database.name + env_dict["CLP_DB_USER"] = clp_config.database.username + env_dict["CLP_DB_PASS"] = clp_config.database.password - if not wait_for_container_cmd(container_name, mysqladmin_cmd, 180): - raise EnvironmentError(f"{component_name} did not initialize in time") - logger.info(f"Started {component_name}.") - - -def create_db_tables( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, -): - component_name = DB_COMPONENT_NAME - logger.info(f"Creating {component_name} tables...") - - container_name = f"clp-{component_name}-table-creator-{instance_id}" - - clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages" - # fmt: off - container_start_cmd = [ - "docker", "run", - "-i", - "--network", "host", - "--rm", - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - env_vars = [ - *get_common_env_vars_list(), - *get_credential_env_vars_list(container_clp_config, include_db_credentials=True), - ] - necessary_mounts = [ - mounts.clp_home, - mounts.data_dir, - mounts.logs_dir, - mounts.generated_config_file, - ] - append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) - - clp_py_utils_dir = clp_site_packages_dir / "clp_py_utils" - # fmt: off - create_tables_cmd = [ - "python3", - str(clp_py_utils_dir / "create-db-tables.py"), - "--config", str(container_clp_config.get_shared_config_file_path()), - "--storage-engine", str(container_clp_config.package.storage_engine), - ] - # fmt: on - - cmd = container_start_cmd + create_tables_cmd - logger.debug(" ".join(cmd)) - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Created {component_name} tables.") - - -def create_results_cache_indices( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, -): - component_name = RESULTS_CACHE_COMPONENT_NAME - logger.info(f"Creating {component_name} indices...") - - container_name = f"clp-{component_name}-indices-creator-{instance_id}" - - clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages" - # fmt: off - container_start_cmd = [ - "docker", "run", - "-i", - "--network", "host", - "--rm", - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - env_vars = [f"PYTHONPATH={clp_site_packages_dir}"] - necessary_mounts = [mounts.clp_home, mounts.data_dir, mounts.logs_dir] - append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) - - clp_py_utils_dir = clp_site_packages_dir / "clp_py_utils" - # fmt: off - init_cmd = [ - "python3", - str(clp_py_utils_dir / "initialize-results-cache.py"), - "--uri", container_clp_config.results_cache.get_uri(), - "--stream-collection", container_clp_config.results_cache.stream_collection_name, - ] - # fmt: on - - cmd = container_start_cmd + init_cmd - logger.debug(" ".join(cmd)) - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Created {component_name} indices.") - - -def start_queue(instance_id: str, clp_config: CLPConfig): +def start_queue(clp_config: CLPConfig): component_name = QUEUE_COMPONENT_NAME - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return + logger.info(f"Initializing {component_name}...") queue_logs_dir = clp_config.logs_directory / component_name validate_queue_config(clp_config, queue_logs_dir) - log_filename = "rabbitmq.log" - - # Generate config file - config_filename = f"{container_name}.conf" - host_config_file_path = clp_config.logs_directory / config_filename - with open(host_config_file_path, "w") as f: - f.write(f"default_user = {clp_config.queue.username}\n") - f.write(f"default_pass = {clp_config.queue.password}\n") - f.write(f"log.file = {log_filename}\n") + env_dict["CLP_QUEUE_HOST"] = get_ip_from_hostname(clp_config.queue.host) + env_dict["CLP_QUEUE_PORT"] = str(clp_config.queue.port) + env_dict["CLP_QUEUE_USER"] = clp_config.queue.username + env_dict["CLP_QUEUE_PASS"] = clp_config.queue.password # Create directories queue_logs_dir.mkdir(exist_ok=True, parents=True) - # Start container - rabbitmq_logs_dir = pathlib.Path("/") / "var" / "log" / "rabbitmq" - mounts = [ - DockerMount( - DockerMountType.BIND, - host_config_file_path, - pathlib.Path("/") / "etc" / "rabbitmq" / "rabbitmq.conf", - True, - ), - DockerMount(DockerMountType.BIND, queue_logs_dir, rabbitmq_logs_dir), - ] - rabbitmq_pid_file_path = pathlib.Path("/") / "tmp" / "rabbitmq.pid" - - host_user_id = os.getuid() - if 0 != host_user_id: - container_user = f"{host_user_id}:{os.getgid()}" - else: - # The host user is `root` so use the container's default user and make this component's - # directories writable by that user. - # NOTE: This doesn't affect the host user's access to the directories since they're `root`. - container_user = "rabbitmq" - default_container_user_id = 999 - default_container_group_id = 999 - chown_recursively(queue_logs_dir, default_container_user_id, default_container_group_id) - - # fmt: off - cmd = [ - "docker", "run", - "-d", - "--name", container_name, - "--log-driver", "local", - # Override RABBITMQ_LOGS since the image sets it to *only* log to stdout - "-u", container_user - ] - env_vars = [ - f"RABBITMQ_LOGS={rabbitmq_logs_dir / log_filename}", - f"RABBITMQ_PID_FILE={rabbitmq_pid_file_path}", - ] - # fmt: on - append_docker_options(cmd, mounts, env_vars) - append_docker_port_settings_for_host_ips( - clp_config.queue.host, clp_config.queue.port, 5672, cmd - ) - cmd.append("rabbitmq:3.9.8") - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - # Wait for queue to start up - rabbitmq_cmd = ["rabbitmq-diagnostics", "check_running"] - if not wait_for_container_cmd(container_name, rabbitmq_cmd, 60): - raise EnvironmentError(f"{component_name} did not initialize in time") - - logger.info(f"Started {component_name}.") + env_dict["CLP_HOST_QUEUE_LOGS_DIR"] = str(queue_logs_dir) -def start_redis(instance_id: str, clp_config: CLPConfig, conf_dir: pathlib.Path): +def start_redis(clp_config: CLPConfig, conf_dir: pathlib.Path): component_name = REDIS_COMPONENT_NAME - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return + logger.info(f"Initializing {component_name}...") redis_logs_dir = clp_config.logs_directory / component_name redis_data_dir = clp_config.data_directory / component_name - base_config_file_path = conf_dir / "redis" / "redis.conf" - validate_redis_config(clp_config, redis_data_dir, redis_logs_dir, base_config_file_path) + config_file_path = conf_dir / "redis" / "redis.conf" + validate_redis_config(clp_config, redis_data_dir, redis_logs_dir, config_file_path) - config_filename = f"{container_name}.conf" - host_config_file_path = clp_config.logs_directory / config_filename - with open(base_config_file_path, "r") as base, open(host_config_file_path, "w") as full: - for line in base.readlines(): - full.write(line) - full.write(f"requirepass {clp_config.redis.password}\n") + env_dict["CLP_HOST_REDIS_CONF_PATH"] = str(config_file_path) + env_dict["CLP_HOST_REDIS_DATA_DIR"] = str(redis_data_dir) + env_dict["CLP_HOST_REDIS_LOGS_DIR"] = str(redis_logs_dir) redis_data_dir.mkdir(exist_ok=True, parents=True) redis_logs_dir.mkdir(exist_ok=True, parents=True) - # Start container - config_file_path = pathlib.Path("/") / "usr" / "local" / "etc" / "redis" / "redis.conf" - mounts = [ - DockerMount(DockerMountType.BIND, host_config_file_path, config_file_path, True), - DockerMount( - DockerMountType.BIND, redis_logs_dir, pathlib.Path("/") / "var" / "log" / "redis" - ), - DockerMount(DockerMountType.BIND, redis_data_dir, pathlib.Path("/") / "data"), - ] - - host_user_id = os.getuid() - if 0 != host_user_id: - container_user = f"{host_user_id}:{os.getgid()}" - else: - # The host user is `root` so use the container's default user and make this component's - # directories writable by that user. - # NOTE: This doesn't affect the host user's access to the directories since they're `root`. - container_user = "redis" - default_container_user_id = 999 - default_container_group_id = 999 - chown_recursively(redis_data_dir, default_container_user_id, default_container_group_id) - chown_recursively(redis_logs_dir, default_container_user_id, default_container_group_id) - - # fmt: off - cmd = [ - "docker", "run", - "-d", - "--name", container_name, - "--log-driver", "local", - "-u", container_user, - ] - # fmt: on - append_docker_options(cmd, mounts) - append_docker_port_settings_for_host_ips( - clp_config.redis.host, clp_config.redis.port, 6379, cmd - ) - cmd.append("redis:7.2.4") - cmd.append("redis-server") - cmd.append(str(config_file_path)) - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - # fmt: off - redis_ping_cmd = [ - "redis-cli", - "-h", "127.0.0.1", - "-p", "6379", - "-a", clp_config.redis.password, - "PING" - ] - # fmt: on - - if not wait_for_container_cmd(container_name, redis_ping_cmd, 30): - raise EnvironmentError(f"{component_name} did not initialize in time") - - logger.info(f"Started {component_name}.") + env_dict["CLP_REDIS_HOST"] = get_ip_from_hostname(clp_config.redis.host) + env_dict["CLP_REDIS_PORT"] = str(clp_config.redis.port) + env_dict["CLP_REDIS_PASS"] = clp_config.redis.password + env_dict["CLP_REDIS_QUERY_BACKEND_DB"] = str(clp_config.redis.query_backend_database) + env_dict["CLP_REDIS_COMPRESSION_BACKEND_DB"] = str( + clp_config.redis.compression_backend_database) -def start_results_cache(instance_id: str, clp_config: CLPConfig, conf_dir: pathlib.Path): +def start_results_cache(clp_config: CLPConfig, conf_dir: pathlib.Path): component_name = RESULTS_CACHE_COMPONENT_NAME - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return + logger.info(f"Initializing {component_name}...") data_dir = clp_config.data_directory / component_name logs_dir = clp_config.logs_directory / component_name @@ -498,298 +246,110 @@ def start_results_cache(instance_id: str, clp_config: CLPConfig, conf_dir: pathl data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) - mounts = [ - DockerMount( - DockerMountType.BIND, conf_dir / "mongo", pathlib.Path("/") / "etc" / "mongo", True - ), - DockerMount(DockerMountType.BIND, data_dir, pathlib.Path("/") / "data" / "db"), - DockerMount(DockerMountType.BIND, logs_dir, pathlib.Path("/") / "var" / "log" / "mongodb"), - ] - - host_user_id = os.getuid() - if 0 != host_user_id: - container_user = f"{host_user_id}:{os.getgid()}" - else: - # The host user is `root` so use the container's default user and make this component's - # directories writable by that user. - # NOTE: This doesn't affect the host user's access to the directories since they're `root`. - container_user = "mongodb" - default_container_user_id = 999 - default_container_group_id = 999 - chown_recursively(data_dir, default_container_user_id, default_container_group_id) - chown_recursively(logs_dir, default_container_user_id, default_container_group_id) - - # fmt: off - cmd = [ - "docker", "run", - "-d", - "--network", "host", - "--name", container_name, - "--log-driver", "local", - "-u", container_user, - ] - # fmt: on - append_docker_options(cmd, mounts) - cmd.append("mongo:7.0.1") - cmd.append("--config") - cmd.append(str(pathlib.Path("/") / "etc" / "mongo" / "mongod.conf")) - cmd.append("--bind_ip") - cmd.append(clp_config.results_cache.host) - cmd.append("--port") - cmd.append(str(clp_config.results_cache.port)) - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Started {component_name}.") + env_dict["CLP_HOST_RESULTS_CACHE_CONF_DIR"] = str(conf_dir / "mongo") + env_dict["CLP_HOST_RESULTS_CACHE_DATA_DIR"] = str(data_dir) + env_dict["CLP_HOST_RESULTS_CACHE_LOGS_DIR"] = str(logs_dir) + + env_dict["CLP_RESULTS_CACHE_HOST"] = get_ip_from_hostname(clp_config.results_cache.host) + env_dict["CLP_RESULTS_CACHE_PORT"] = str(clp_config.results_cache.port) + env_dict["CLP_RESULTS_CACHE_DB_NAME"] = clp_config.results_cache.db_name + env_dict[ + "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME"] = clp_config.results_cache.stream_collection_name def start_compression_scheduler( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, + clp_config: CLPConfig, ): - module_name = "job_orchestration.scheduler.compress.compression_scheduler" - generic_start_scheduler( - COMPRESSION_SCHEDULER_COMPONENT_NAME, - module_name, - instance_id, - clp_config, - container_clp_config, - mounts, - ) + component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + logs_dir = clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) -def start_query_scheduler( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, -): - module_name = "job_orchestration.scheduler.query.query_scheduler" - generic_start_scheduler( - QUERY_SCHEDULER_COMPONENT_NAME, - module_name, - instance_id, - clp_config, - container_clp_config, - mounts, - ) + env_dict["CLP_HOST_COMPRESSION_SCHEDULER_LOGS_DIR"] = str(logs_dir) + env_dict[ + "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL"] = clp_config.compression_scheduler.logging_level + # FIXME: handle S3 -def generic_start_scheduler( - component_name: str, - module_name: str, - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, -): - logger.info(f"Starting {component_name}...") - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return +def start_query_scheduler( + clp_config: CLPConfig, +): + component_name = QUERY_SCHEDULER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - container_logs_dir = container_clp_config.logs_directory / component_name - - clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages" - # fmt: off - container_start_cmd = [ - "docker", "run", - "-di", - "--network", "host", - "-w", str(CONTAINER_CLP_HOME), - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - - env_vars = [ - *get_common_env_vars_list(), - *get_credential_env_vars_list(container_clp_config, include_db_credentials=True), - *get_celery_connection_env_vars_list(container_clp_config), - f"CLP_LOGS_DIR={container_logs_dir}", - f"CLP_LOGGING_LEVEL={clp_config.query_scheduler.logging_level}", - ] - necessary_mounts = [mounts.clp_home, mounts.logs_dir, mounts.generated_config_file] - aws_mount, aws_env_vars = generate_container_auth_options(clp_config, component_name) - if aws_mount: - necessary_mounts.append(mounts.aws_config_dir) - if aws_env_vars: - env_vars.extend(aws_env_vars) - if ( - COMPRESSION_SCHEDULER_COMPONENT_NAME == component_name - and StorageType.FS == clp_config.logs_input.type - ): - necessary_mounts.append(mounts.input_logs_dir) - append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) - - # fmt: off - scheduler_cmd = [ - "python3", "-u", - "-m", module_name, - "--config", str(container_clp_config.get_shared_config_file_path()), - ] - # fmt: on - cmd = container_start_cmd + scheduler_cmd - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Started {component_name}.") + + env_dict["CLP_HOST_QUERY_SCHEDULER_LOGS_DIR"] = str(logs_dir) + env_dict["CLP_QUERY_SCHEDULER_LOGGING_LEVEL"] = clp_config.query_scheduler.logging_level + + # FIXME: handle S3 def start_compression_worker( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - num_cpus: int, - mounts: CLPDockerMounts, + clp_config: CLPConfig, + num_cpus: int, ): - celery_method = "job_orchestration.executor.compress" - celery_route = f"{QueueName.COMPRESSION}" - compression_worker_mounts = [mounts.archives_output_dir] - generic_start_worker( - COMPRESSION_WORKER_COMPONENT_NAME, - instance_id, - clp_config, - clp_config.compression_worker, - container_clp_config, - celery_method, - celery_route, - clp_config.redis.compression_backend_database, - num_cpus, - mounts, - compression_worker_mounts, - ) + component_name = COMPRESSION_WORKER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + logs_dir = clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) -def start_query_worker( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - num_cpus: int, - mounts: CLPDockerMounts, -): - celery_method = "job_orchestration.executor.query" - celery_route = f"{QueueName.QUERY}" - - query_worker_mounts = [mounts.stream_output_dir] - if StorageType.FS == clp_config.archive_output.storage.type: - query_worker_mounts.append(mounts.archives_output_dir) - - generic_start_worker( - QUERY_WORKER_COMPONENT_NAME, - instance_id, - clp_config, - clp_config.query_worker, - container_clp_config, - celery_method, - celery_route, - clp_config.redis.query_backend_database, - num_cpus, - mounts, - query_worker_mounts, - ) + env_dict["CLP_COMPRESSION_WORKER_LOGS_DIR"] = str(logs_dir) + env_dict["CLP_COMPRESSION_WORKER_LOGGING_LEVEL"] = clp_config.compression_worker.logging_level + env_dict["CLP_COMPRESSION_WORKER_CONCURRENCY"] = str(num_cpus) + # Create necessary directories + clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) + clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) + + # FIXME: handle S3 -def generic_start_worker( - component_name: str, - instance_id: str, - clp_config: CLPConfig, - worker_config: BaseModel, - container_clp_config: CLPConfig, - celery_method: str, - celery_route: str, - redis_database: int, - num_cpus: int, - mounts: CLPDockerMounts, - worker_specific_mounts: Optional[List[Optional[DockerMount]]], -): - logger.info(f"Starting {component_name}...") - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return +def start_query_worker( + clp_config: CLPConfig, + num_cpus: int, +): + component_name = QUERY_WORKER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - container_logs_dir = container_clp_config.logs_directory / component_name + + env_dict["CLP_QUERY_WORKER_LOGS_DIR"] = str(logs_dir) + env_dict["CLP_QUERY_WORKER_LOGGING_LEVEL"] = clp_config.query_worker.logging_level + env_dict["CLP_QUERY_WORKER_CONCURRENCY"] = str(num_cpus) # Create necessary directories clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) - clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages" - container_worker_log_path = container_logs_dir / "worker.log" - # fmt: off - container_start_cmd = [ - "docker", "run", - "-di", - "--network", "host", - "-w", str(CONTAINER_CLP_HOME), - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - - env_vars = [ - *get_common_env_vars_list(include_clp_home_env_var=True), - *get_celery_connection_env_vars_list(container_clp_config), - f"CLP_CONFIG_PATH={container_clp_config.get_shared_config_file_path()}", - f"CLP_LOGS_DIR={container_logs_dir}", - f"CLP_LOGGING_LEVEL={worker_config.logging_level}", - f"CLP_WORKER_LOG_PATH={container_worker_log_path}", - ] - necessary_mounts = [ - mounts.clp_home, - mounts.data_dir, - mounts.logs_dir, - ] - if StorageType.FS == clp_config.logs_input.type: - necessary_mounts.append(mounts.input_logs_dir) - if worker_specific_mounts: - necessary_mounts.extend(worker_specific_mounts) - - aws_mount, aws_env_vars = generate_container_auth_options(clp_config, component_name) - if aws_mount: - necessary_mounts.append(mounts.aws_config_dir) - if aws_env_vars: - env_vars.extend(aws_env_vars) - - append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) - - worker_cmd = [ - "python3", - str(clp_site_packages_dir / "bin" / "celery"), - "-A", - celery_method, - "worker", - "--concurrency", - str(num_cpus), - "--loglevel", - "WARNING", - "-f", - str(container_worker_log_path), - "-Q", - celery_route, - "-n", - component_name, - ] - cmd = container_start_cmd + worker_cmd - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Started {component_name}.") + # FIXME: handle S3 + + +def start_reducer( + clp_config: CLPConfig, + num_workers: int, +): + component_name = REDUCER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_dir = clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) + + env_dict["CLP_REDUCER_LOGS_DIR"] = str(logs_dir) + env_dict["CLP_REDUCER_LOGGING_LEVEL"] = clp_config.reducer.logging_level + env_dict["CLP_REDUCER_CONCURRENCY"] = str(num_workers) + env_dict["CLP_REDUCER_UPSERT_INTERVAL"] = str(clp_config.reducer.upsert_interval) def update_settings_object( - parent_key_prefix: str, - settings: Dict[str, Any], - updates: Dict[str, Any], + parent_key_prefix: str, + settings: Dict[str, Any], + updates: Dict[str, Any], ): """ Recursively updates the given settings object with the values from `updates`. @@ -824,25 +384,18 @@ def read_and_update_settings_json(settings_file_path: pathlib.Path, updates: Dic def start_webui( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, + clp_config: CLPConfig, + container_clp_config: CLPConfig, ): component_name = WEBUI_COMPONENT_NAME - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return + logger.info(f"Initializing {component_name}...") container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" - node_path = str(container_webui_dir / "server" / "node_modules") client_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "client" / "settings.json" + get_clp_home() / "var" / "www" / "webui" / "client" / "settings.json" ) server_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "server" / "settings.json" + get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "server" / "settings.json" ) validate_webui_config(clp_config, client_settings_json_path, server_settings_json_path) @@ -874,22 +427,20 @@ def start_webui( client_settings_json_file.write(json.dumps(client_settings_json)) server_settings_json_updates = { - "SqlDbHost": clp_config.database.host, + "SqlDbHost": container_clp_config.database.host, "SqlDbPort": clp_config.database.port, "SqlDbName": clp_config.database.name, "SqlDbQueryJobsTableName": QUERY_JOBS_TABLE_NAME, - "MongoDbHost": clp_config.results_cache.host, + "MongoDbHost": container_clp_config.results_cache.host, "MongoDbPort": clp_config.results_cache.port, "MongoDbName": clp_config.results_cache.db_name, "MongoDbSearchResultsMetadataCollectionName": clp_config.webui.results_metadata_collection_name, "MongoDbStreamFilesCollectionName": clp_config.results_cache.stream_collection_name, "ClientDir": str(container_webui_dir / "client"), "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), - "StreamTargetUncompressedSize": container_clp_config.stream_output.target_uncompressed_size, + "StreamTargetUncompressedSize": clp_config.stream_output.target_uncompressed_size, } - container_cmd_extra_opts = [] - stream_storage = clp_config.stream_output.storage if StorageType.S3 == stream_storage.type: s3_config = stream_storage.s3_config @@ -917,193 +468,22 @@ def start_webui( with open(server_settings_json_path, "w") as settings_json_file: settings_json_file.write(json.dumps(server_settings_json)) - # fmt: off - container_cmd = [ - "docker", "run", - "-d", - "--network", "host", - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - container_cmd.extend(container_cmd_extra_opts) - - env_vars = [ - *get_common_env_vars_list(), - *get_credential_env_vars_list(container_clp_config, include_db_credentials=True), - f"NODE_PATH={node_path}", - f"HOST={clp_config.webui.host}", - f"PORT={clp_config.webui.port}", - f"NODE_ENV=production", - ] - necessary_mounts = [ - mounts.clp_home, - ] - if StorageType.S3 == stream_storage.type: - auth = stream_storage.s3_config.aws_authentication - if AwsAuthType.credentials == auth.type: - credentials = auth.credentials - env_vars.append(f"AWS_ACCESS_KEY_ID={credentials.access_key_id}") - env_vars.append(f"AWS_SECRET_ACCESS_KEY={credentials.secret_access_key}") - else: - aws_mount, aws_env_vars = generate_container_auth_options( - clp_config, WEBUI_COMPONENT_NAME - ) - if aws_mount: - necessary_mounts.append(mounts.aws_config_dir) - if aws_env_vars: - env_vars.extend(aws_env_vars) - elif StorageType.FS == stream_storage.type: - necessary_mounts.append(mounts.stream_output_dir) - - append_docker_options(container_cmd, necessary_mounts, env_vars) - container_cmd.append(clp_config.execution_container) - - node_cmd = [ - str(CONTAINER_CLP_HOME / "bin" / "node-22"), - str(container_webui_dir / "server" / "dist" / "server" / "src" / "main.js"), - ] - cmd = container_cmd + node_cmd - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Started {component_name}.") - - -def start_reducer( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - num_workers: int, - mounts: CLPDockerMounts, -): - component_name = REDUCER_COMPONENT_NAME - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return - - logs_dir = clp_config.logs_directory / component_name - validate_reducer_config(clp_config, logs_dir, num_workers) - - logs_dir.mkdir(parents=True, exist_ok=True) - container_logs_dir = container_clp_config.logs_directory / component_name - - clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages" - # fmt: off - container_start_cmd = [ - "docker", "run", - "-di", - "--network", "host", - "-w", str(CONTAINER_CLP_HOME), - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - env_vars = [ - *get_common_env_vars_list(include_clp_home_env_var=True), - f"CLP_LOGS_DIR={container_logs_dir}", - f"CLP_LOGGING_LEVEL={clp_config.reducer.logging_level}", - ] - necessary_mounts = [ - mounts.clp_home, - mounts.logs_dir, - mounts.generated_config_file, - ] - append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) - - # fmt: off - reducer_cmd = [ - "python3", "-u", - "-m", "job_orchestration.reducer.reducer", - "--config", str(container_clp_config.get_shared_config_file_path()), - "--concurrency", str(num_workers), - "--upsert-interval", str(clp_config.reducer.upsert_interval), - ] - # fmt: on - cmd = container_start_cmd + reducer_cmd - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Started {component_name}.") + env_dict["CLP_WEBUI_HOST"] = get_ip_from_hostname(clp_config.webui.host) + env_dict["CLP_WEBUI_PORT"] = clp_config.webui.port + # FIXME: Handle S3 def start_garbage_collector( - instance_id: str, clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, ): component_name = GARBAGE_COLLECTOR_COMPONENT_NAME - - if not is_retention_period_configured(clp_config): - logger.info(f"Retention period is not configured, skipping {component_name} creation...") - return - - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return + logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name - validate_log_directory(logs_dir, component_name) - # Create logs directory if necessary logs_dir.mkdir(parents=True, exist_ok=True) - container_logs_dir = container_clp_config.logs_directory / component_name - - # fmt: off - container_start_cmd = [ - "docker", "run", - "-di", - "--network", "host", - "-w", str(CONTAINER_CLP_HOME), - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - - necessary_mounts = [ - mounts.clp_home, - mounts.logs_dir, - mounts.generated_config_file, - ] - env_vars = [ - *get_common_env_vars_list(include_clp_home_env_var=True), - *get_credential_env_vars_list(container_clp_config, include_db_credentials=True), - f"CLP_LOGS_DIR={container_logs_dir}", - f"CLP_LOGGING_LEVEL={clp_config.garbage_collector.logging_level}", - ] - - # Add necessary mounts for archives and streams. - if StorageType.FS == clp_config.archive_output.storage.type: - necessary_mounts.append(mounts.archives_output_dir) - if StorageType.FS == clp_config.stream_output.storage.type: - necessary_mounts.append(mounts.stream_output_dir) - - aws_mount, aws_env_vars = generate_container_auth_options(clp_config, component_name) - if aws_mount: - necessary_mounts.append(mounts.aws_config_dir) - if aws_env_vars: - env_vars.extend(aws_env_vars) - - append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) - - # fmt: off - garbage_collector_cmd = [ - "python3", "-u", - "-m", "job_orchestration.garbage_collector.garbage_collector", - "--config", str(container_clp_config.get_shared_config_file_path()), - ] - # fmt: on - cmd = container_start_cmd + garbage_collector_cmd - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Started {component_name}.") + + env_dict["CLP_GC_LOGS_DIR"] = str(logs_dir) + env_dict["CLP_GC_LOGGING_LEVEL"] = clp_config.garbage_collector.logging_level def add_num_workers_argument(parser): @@ -1113,6 +493,7 @@ def add_num_workers_argument(parser): default=multiprocessing.cpu_count(), help="Number of workers to start", ) + # FIXME: handle S3 def main(argv): @@ -1146,11 +527,6 @@ def main(argv): parsed_args = args_parser.parse_args(argv[1:]) - if parsed_args.target: - target = parsed_args.target - else: - target = ALL_TARGET_NAME - try: check_dependencies() except: @@ -1162,64 +538,17 @@ def main(argv): config_file_path = pathlib.Path(parsed_args.config) clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) - runnable_components = clp_config.get_runnable_components() - components_to_start = get_components_for_target(target) - components_to_start = components_to_start.intersection(runnable_components) - - # Exit early if no components to start - if len(components_to_start) == 0: - logger.error(f"{target} not available with current configuration") - return -1 - - # Validate and load necessary credentials - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - DB_COMPONENT_NAME, - GARBAGE_COLLECTOR_COMPONENT_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - WEBUI_COMPONENT_NAME, - ): - validate_and_load_db_credentials_file(clp_config, clp_home, True) - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - QUEUE_COMPONENT_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - ): - validate_and_load_queue_credentials_file(clp_config, clp_home, True) - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - REDIS_COMPONENT_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - ): - validate_and_load_redis_credentials_file(clp_config, clp_home, True) - if target in ( - ALL_TARGET_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - ): - validate_logs_input_config(clp_config) - if target in ( - ALL_TARGET_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - GARBAGE_COLLECTOR_COMPONENT_NAME, - ): - validate_output_storage_config(clp_config) - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - GARBAGE_COLLECTOR_COMPONENT_NAME, - ): - validate_retention_config(clp_config) + validate_and_load_db_credentials_file(clp_config, clp_home, True) + validate_and_load_queue_credentials_file(clp_config, clp_home, True) + validate_and_load_redis_credentials_file(clp_config, clp_home, True) + validate_logs_input_config(clp_config) + validate_output_storage_config(clp_config) + validate_retention_config(clp_config) + + # validate_and_load_db_credentials_file(clp_config, clp_home, True) + # validate_and_load_queue_credentials_file(clp_config, clp_home, True) + # validate_and_load_redis_credentials_file(clp_config, clp_home, True) + # validate_worker_config(clp_config) clp_config.validate_data_dir() clp_config.validate_logs_dir() @@ -1228,14 +557,8 @@ def main(argv): logger.exception("Failed to load config.") return -1 - if target in ( - COMPRESSION_WORKER_COMPONENT_NAME, - REDUCER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - ): - num_workers = parsed_args.num_workers - else: - num_workers = multiprocessing.cpu_count() // 2 + # FIXME: handle this + num_workers = multiprocessing.cpu_count() // 2 container_clp_config, mounts = generate_container_config(clp_config, clp_home) @@ -1243,75 +566,62 @@ def main(argv): clp_config.data_directory.mkdir(parents=True, exist_ok=True) clp_config.logs_directory.mkdir(parents=True, exist_ok=True) + # Set container services' hosts in container config + container_clp_config.database.host = DB_COMPONENT_NAME + container_clp_config.queue.host = QUEUE_COMPONENT_NAME + container_clp_config.redis.host = REDIS_COMPONENT_NAME + container_clp_config.results_cache.host = RESULTS_CACHE_COMPONENT_NAME + container_clp_config.query_scheduler.host = QUERY_SCHEDULER_COMPONENT_NAME + container_clp_config.reducer.host = REDUCER_COMPONENT_NAME + dump_shared_container_config(container_clp_config, clp_config) - try: - # Create instance-id file - instance_id_file_path = clp_config.logs_directory / "instance-id" - if instance_id_file_path.exists(): - with open(instance_id_file_path, "r") as f: - instance_id = f.readline() - else: - instance_id = str(uuid.uuid4())[-4:] - with open(instance_id_file_path, "w") as f: - f.write(instance_id) - f.flush() + env_dict["CLP_USER_ID"] = os.getuid() + env_dict["CLP_GROUP_ID"] = os.getgid() + env_dict["CLP_STORAGE_ENGINE"] = clp_config.package.storage_engine + + env_dict["CLP_HOST_DATA_DIR"] = str(clp_config.data_directory) + env_dict["CLP_HOST_LOGS_DIR"] = str(clp_config.logs_directory) + env_dict["CLP_HOST_ARCHIVE_OUTPUT_DIR"] = str(clp_config.archive_output.get_directory()) + env_dict["CLP_HOST_STREAM_OUTPUT_DIR"] = str(clp_config.stream_output.get_directory()) + try: conf_dir = clp_home / "etc" # Start components - if DB_COMPONENT_NAME in components_to_start: - start_db(instance_id, clp_config, conf_dir) - - if ( - target == CONTROLLER_TARGET_NAME and DB_COMPONENT_NAME in runnable_components - ) or DB_COMPONENT_NAME in components_to_start: - create_db_tables(instance_id, clp_config, container_clp_config, mounts) - - if QUEUE_COMPONENT_NAME in components_to_start: - start_queue(instance_id, clp_config) - - if REDIS_COMPONENT_NAME in components_to_start: - start_redis(instance_id, clp_config, conf_dir) - - if RESULTS_CACHE_COMPONENT_NAME in components_to_start: - start_results_cache(instance_id, clp_config, conf_dir) - - if ( - target == CONTROLLER_TARGET_NAME and RESULTS_CACHE_COMPONENT_NAME in runnable_components - ) or RESULTS_CACHE_COMPONENT_NAME in components_to_start: - create_results_cache_indices(instance_id, clp_config, container_clp_config, mounts) - - if COMPRESSION_SCHEDULER_COMPONENT_NAME in components_to_start: - start_compression_scheduler(instance_id, clp_config, container_clp_config, mounts) - - if QUERY_SCHEDULER_COMPONENT_NAME in components_to_start: - start_query_scheduler(instance_id, clp_config, container_clp_config, mounts) - - if COMPRESSION_WORKER_COMPONENT_NAME in components_to_start: - start_compression_worker( - instance_id, clp_config, container_clp_config, num_workers, mounts - ) - - if QUERY_WORKER_COMPONENT_NAME in components_to_start: - start_query_worker(instance_id, clp_config, container_clp_config, num_workers, mounts) - - if REDUCER_COMPONENT_NAME in components_to_start: - start_reducer(instance_id, clp_config, container_clp_config, num_workers, mounts) - - if WEBUI_COMPONENT_NAME in components_to_start: - start_webui(instance_id, clp_config, container_clp_config, mounts) - - if GARBAGE_COLLECTOR_COMPONENT_NAME in components_to_start: - start_garbage_collector(instance_id, clp_config, container_clp_config, mounts) - + start_db( clp_config, conf_dir) + start_queue( clp_config) + start_redis( clp_config, conf_dir) + start_results_cache( clp_config, conf_dir) + start_compression_scheduler(clp_config) + start_query_scheduler(clp_config) + start_compression_worker(clp_config, num_workers) + start_query_worker( clp_config, num_workers) + start_reducer( clp_config, num_workers) + start_webui( clp_config, container_clp_config) + start_garbage_collector(clp_config) + + with open(f"{clp_home}/.env", "w") as env_file: + for key, value in env_dict.items(): + env_file.write(f"{key}={value}\n") except Exception as ex: if type(ex) == ValueError: - logger.error(f"Failed to start CLP: {ex}") + logger.error(f"Failed to initialize CLP: {ex}") else: - logger.exception("Failed to start CLP.") + logger.exception("Failed to initialize CLP.") return -1 + # try: + # subprocess.run( + # "docker compose up", + # shell=True, + # stderr=subprocess.STDOUT, + # check=True, + # ) + # except subprocess.CalledProcessError: + # logger.exception("Failed to start CLP.") + # return -1 + return 0 diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml new file mode 100644 index 0000000000..1bfd80d95e --- /dev/null +++ b/tools/deployment/package/docker-compose.yml @@ -0,0 +1,374 @@ +secrets: + CLP_DB_PASS_FILE: + environment: "CLP_DB_PASS" +services: + db: + image: ${CLP_DB_IMAGE:-mysql:8.0.23} + container_name: "database" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + secrets: + - CLP_DB_PASS_FILE + environment: + MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_USER: "${CLP_DB_USER}" + MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_DATABASE: "${CLP_DB_NAME}" + volumes: + - "./etc/mysql/conf.d/logging.cnf:/etc/mysql/conf.d/logging.cnf:ro" + - "./var/data/database:/var/lib/mysql" + - "./var/log/database:/var/log/mysql" + networks: + - "clp-network" + ports: + - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" + healthcheck: + test: [ + "CMD", + "mysqladmin", "ping", + "--silent", + "-h", "127.0.0.1", + "-u", "${CLP_DB_USER}", + "--password=${CLP_DB_PASS}" + ] + start_period: "10s" + start_interval: "2s" + interval: "30s" + timeout: "10s" + retries: 3 + + db-table-creator: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "db_table_creator" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS}" + command: [ + "python3", + "-u", + "-m", "clp_py_utils.create-db-tables", + "--config", "/etc/clp-config.yml", + "--storage-engine", "${CLP_STORAGE_ENGINE}" + ] + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml" + networks: + - "clp-network" + depends_on: + db: + condition: "service_healthy" + + queue: + image: "rabbitmq:3.9.8" + container_name: "queue" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" + RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" + RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" + volumes: + - "./var/log/queue:/var/log/rabbitmq" + networks: + - "clp-network" + ports: + - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" + healthcheck: + test: [ + "CMD", + "rabbitmq-diagnostics", "check_running" + ] + start_period: "10s" + start_interval: "2s" + interval: "30s" + timeout: "10s" + retries: 3 + + redis: + image: "redis:7.2.4" + container_name: "redis" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + command: [ + "redis-server", + "/usr/local/etc/redis/redis.conf", + "--requirepass", "${CLP_REDIS_PASS}" + ] + volumes: + - "./etc/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro" + - "./var/log/redis:/var/log/redis" + - "./var/data/redis:/data" + networks: + - "clp-network" + ports: + - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" + healthcheck: + test: [ + "CMD", + "redis-cli", + "-h", "127.0.0.1", + "-p", "6379", + "-a", "${CLP_REDIS_PASS}", + "PING" + ] + start_period: "10s" + start_interval: "2s" + interval: 30s + timeout: 10s + retries: 3 + + results-cache: + image: "mongo:7.0.1" + container_name: "results_cache" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + command: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0", + ] + volumes: + - "./etc/mongo:/etc/mongo:ro" + - "./var/data/results_cache:/data/db" + - "./var/log/results_cache:/var/log/mongodb" + networks: + - "clp-network" + ports: + - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet + start_period: "10s" + start_interval: "2s" + interval: 30s + timeout: 10s + retries: 3 + + results-cache-indices-creator: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "results_cache_indices_creator" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + command: [ + "python3", + "-u", + "-m", "clp_py_utils.initialize-results-cache", + "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", + "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", + ] + networks: + - "clp-network" + depends_on: + results-cache: + condition: "service_healthy" + + compression-scheduler: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "compression_scheduler" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1}" + CLP_LOGS_DIR: "/var/log" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS}" + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "./var/log:/var/log" + # FIXME: why not call this /mnt/input? + - "/:/mnt/logs:ro" + networks: + - "clp-network" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + queue: + condition: "service_healthy" + redis: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "job_orchestration.scheduler.compress.compression_scheduler", + "--config", "/etc/clp-config.yml" + ] + healthcheck: + # FIXME + test: [ + "CMD", + "true" + ] + + query-scheduler: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "query_scheduler" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0}" + CLP_LOGS_DIR: "/var/log" + CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS}" + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "./var/log:/var/log" + networks: + - "clp-network" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + queue: + condition: "service_healthy" + redis: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "job_orchestration.scheduler.query.query_scheduler", + "--config", "/etc/clp-config.yml" + ] + healthcheck: + # FIXME + test: [ + "CMD", + "true" + ] + + compression-worker: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "compression_worker" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1}" + CLP_HOME: "/opt/clp" + CLP_LOGS_DIR: "/var/log/compression_worker" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" + CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" + CLP_CONFIG_PATH: "/etc/clp-config.yml" + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "./var/log/compression_worker:/var/log/compression_worker" + # FIXME: why dump "var/data/compression-job-4-task-4-db-config.yml"? + - "./var/data:/var/data" + - "./var/data/archives:/var/data/archives" + # FIXME: why not call this /mnt/input? + - "/:/mnt/logs:ro" + networks: + - "clp-network" + depends_on: + compression-scheduler: + condition: "service_healthy" + # FIXME: add healthcheck + command: [ + "python3", + "-u", + "/opt/clp/lib/python3/site-packages/bin/celery", + "-A", "job_orchestration.executor.compress", + "worker", + "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", + "--loglevel", "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}", + "-f", "/var/log/compression_worker/worker.log", + "-Q", "compression", + "-n", "compression-worker" + ] + + query-worker: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "query_worker" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0}" + CLP_HOME: "/opt/clp" + CLP_LOGS_DIR: "/var/log/query_worker" + CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" + CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" + CLP_CONFIG_PATH: "/etc/clp-config.yml" + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "./var/log/query_worker:/var/log/query_worker" + - "./var/data/archives:/var/data/archives" + - "./var/data/streams:/var/data/streams" + networks: + - "clp-network" + depends_on: + query-scheduler: + condition: "service_healthy" + command: [ + "python3", + "-u", + "/opt/clp/lib/python3/site-packages/bin/celery", + "-A", "job_orchestration.executor.query", + "worker", + "--concurrency", "${CLP_QUERY_WORKER_CONCURRENCY:-1}", + "--loglevel", "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}", + "-f", "/var/log/query_worker/worker.log", + "-Q", "query", + "-n", "query-worker" + ] + + reducer: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "reducer" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + CLP_HOME: "/opt/clp" + CLP_LOGS_DIR: "/var/log/reducer" + CLP_LOGGING_LEVEL: "${CLP_REDUCER_LOGGING_LEVEL:-INFO}" + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "./var/log/reducer:/var/log/reducer" + networks: + - "clp-network" + depends_on: + query-scheduler: + condition: "service_healthy" + results-cache-indices-creator: + condition: "service_completed_successfully" + command: [ + "python3", "-u", + "-m", "job_orchestration.reducer.reducer", + "--config", "/etc/clp-config.yml", + "--concurrency", "${CLP_REDUCER_CONCURRENCY:-1}", + "--upsert-interval", "${CLP_REDUCER_UPSERT_INTERVAL:-100}" + ] + + webui: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "webui" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" + HOST: "0.0.0.0" + PORT: "4000" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS}" + NODE_ENV: "production" + volumes: + - "./var/data/streams:/var/data/streams" + - type: "bind" + source: "./var/www/webui/server/dist/server/settings.json" + target: "/opt/clp/var/www/webui/server/dist/server/settings.json" + read_only: true + networks: + - "clp-network" + ports: + - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" + command: [ + "/opt/clp/bin/node-22", + "/opt/clp/var/www/webui/server/dist/server/src/main.js" + ] + depends_on: + db-table-creator: + condition: "service_completed_successfully" + results-cache-indices-creator: + condition: "service_completed_successfully" + +networks: + clp-network: + driver: bridge \ No newline at end of file From 32ea452622a05e6e4f4c90c3436cf68c0bfc7c43 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 22 Aug 2025 04:27:49 -0400 Subject: [PATCH 002/238] Add garbage collector service to Docker Compose configuration --- tools/deployment/package/docker-compose.yml | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 1bfd80d95e..03410db3f2 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -369,6 +369,35 @@ services: results-cache-indices-creator: condition: "service_completed_successfully" + garbage-collector: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "garbage_collector" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + CLP_HOME: "/opt/clp" + CLP_LOGS_DIR: "/var/log/garbage_collector" + CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS}" + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_GC_LOGS_DIR}:/var/log/garbage_collector" + networks: + - "clp-network" + depends_on: + query-scheduler: + condition: "service_healthy" + db-table-creator: + condition: "service_completed_successfully" + results-cache-indices-creator: + condition: "service_completed_successfully" + command: [ + "python3", "-u", + "-m", "job_orchestration.garbage_collector.garbage_collector", + "--config", "/etc/clp-config.yml", + ] + networks: clp-network: driver: bridge \ No newline at end of file From 217c63f4756f0be44cb513d8691f1dc680569294 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 08:36:52 -0400 Subject: [PATCH 003/238] fix s3 support and various issues --- .../clp_package_utils/general.py | 56 ++++++++- .../clp_package_utils/scripts/start_clp.py | 83 +++++++------ .../clp-py-utils/clp_py_utils/clp_config.py | 8 +- tools/deployment/package/docker-compose.yml | 116 +++++++++--------- 4 files changed, 167 insertions(+), 96 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 5191caee23..b27c62606c 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -14,9 +14,11 @@ import yaml from clp_py_utils.clp_config import ( CLP_DEFAULT_CREDENTIALS_FILE_PATH, + CLP_DEFAULT_DATA_DIRECTORY_PATH, CLP_SHARED_CONFIG_FILENAME, CLPConfig, DB_COMPONENT_NAME, + QUERY_SCHEDULER_COMPONENT_NAME, QueryEngine, QUEUE_COMPONENT_NAME, REDIS_COMPONENT_NAME, @@ -152,7 +154,10 @@ def check_dependencies(): raise EnvironmentError("docker cannot run without superuser privileges (sudo).") try: subprocess.run( - ["docker", "compose", "version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True + ["docker", "compose", "version"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True, ) except subprocess.CalledProcessError: raise EnvironmentError("docker-compose is not installed") @@ -316,6 +321,55 @@ def generate_container_config( return container_clp_config, docker_mounts +def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig: + """ + Copies the given config and corrects mount paths and hosts for Docker Compose. + + :param clp_config: + :return: The container config and the mounts. + """ + container_clp_config = clp_config.copy(deep=True) + + # FIXME: consider removing credentials_file_path + + # Set container paths + container_clp_config.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH + container_clp_config.logs_directory = pathlib.Path("/") / "var" / "log" + if StorageType.FS == clp_config.logs_input.type: + container_clp_config.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR + + if StorageType.FS == clp_config.archive_output.storage.type: + container_clp_config.archive_output.storage.directory = ( + pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" + ) + elif StorageType.S3 == clp_config.archive_output.storage.type: + container_clp_config.archive_output.storage.staging_directory = ( + CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" + ) + + if StorageType.FS == clp_config.stream_output.storage.type: + container_clp_config.stream_output.storage.directory = ( + pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" + ) + elif StorageType.S3 == clp_config.stream_output.storage.type: + container_clp_config.stream_output.storage.staging_directory = ( + CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" + ) + + if clp_config.aws_config_directory is not None: + container_clp_config.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY + + # Set container services' hosts + container_clp_config.database.host = DB_COMPONENT_NAME + container_clp_config.queue.host = QUEUE_COMPONENT_NAME + container_clp_config.redis.host = REDIS_COMPONENT_NAME + container_clp_config.results_cache.host = RESULTS_CACHE_COMPONENT_NAME + container_clp_config.query_scheduler.host = QUERY_SCHEDULER_COMPONENT_NAME + container_clp_config.reducer.host = REDUCER_COMPONENT_NAME + + return container_clp_config + + def generate_worker_config(clp_config: CLPConfig) -> WorkerConfig: worker_config = WorkerConfig() worker_config.package = clp_config.package.copy(deep=True) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 1938211ca0..c163df9e0a 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -51,6 +51,7 @@ DockerMountType, dump_shared_container_config, generate_container_config, + generate_docker_compose_container_config, get_celery_connection_env_vars_list, get_clp_home, get_common_env_vars_list, @@ -77,6 +78,7 @@ logger = logging.getLogger(__file__) env_dict = {} + def get_ip_from_hostname(hostname: str) -> str: return socket.gethostbyname(hostname) @@ -231,7 +233,8 @@ def start_redis(clp_config: CLPConfig, conf_dir: pathlib.Path): env_dict["CLP_REDIS_PASS"] = clp_config.redis.password env_dict["CLP_REDIS_QUERY_BACKEND_DB"] = str(clp_config.redis.query_backend_database) env_dict["CLP_REDIS_COMPRESSION_BACKEND_DB"] = str( - clp_config.redis.compression_backend_database) + clp_config.redis.compression_backend_database + ) def start_results_cache(clp_config: CLPConfig, conf_dir: pathlib.Path): @@ -253,12 +256,13 @@ def start_results_cache(clp_config: CLPConfig, conf_dir: pathlib.Path): env_dict["CLP_RESULTS_CACHE_HOST"] = get_ip_from_hostname(clp_config.results_cache.host) env_dict["CLP_RESULTS_CACHE_PORT"] = str(clp_config.results_cache.port) env_dict["CLP_RESULTS_CACHE_DB_NAME"] = clp_config.results_cache.db_name - env_dict[ - "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME"] = clp_config.results_cache.stream_collection_name + env_dict["CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME"] = ( + clp_config.results_cache.stream_collection_name + ) def start_compression_scheduler( - clp_config: CLPConfig, + clp_config: CLPConfig, ): component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") @@ -267,14 +271,15 @@ def start_compression_scheduler( logs_dir.mkdir(parents=True, exist_ok=True) env_dict["CLP_HOST_COMPRESSION_SCHEDULER_LOGS_DIR"] = str(logs_dir) - env_dict[ - "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL"] = clp_config.compression_scheduler.logging_level + env_dict["CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL"] = ( + clp_config.compression_scheduler.logging_level + ) # FIXME: handle S3 def start_query_scheduler( - clp_config: CLPConfig, + clp_config: CLPConfig, ): component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") @@ -289,8 +294,8 @@ def start_query_scheduler( def start_compression_worker( - clp_config: CLPConfig, - num_cpus: int, + clp_config: CLPConfig, + num_cpus: int, ): component_name = COMPRESSION_WORKER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") @@ -310,8 +315,8 @@ def start_compression_worker( def start_query_worker( - clp_config: CLPConfig, - num_cpus: int, + clp_config: CLPConfig, + num_cpus: int, ): component_name = QUERY_WORKER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") @@ -331,8 +336,8 @@ def start_query_worker( def start_reducer( - clp_config: CLPConfig, - num_workers: int, + clp_config: CLPConfig, + num_workers: int, ): component_name = REDUCER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") @@ -347,9 +352,9 @@ def start_reducer( def update_settings_object( - parent_key_prefix: str, - settings: Dict[str, Any], - updates: Dict[str, Any], + parent_key_prefix: str, + settings: Dict[str, Any], + updates: Dict[str, Any], ): """ Recursively updates the given settings object with the values from `updates`. @@ -384,18 +389,18 @@ def read_and_update_settings_json(settings_file_path: pathlib.Path, updates: Dic def start_webui( - clp_config: CLPConfig, - container_clp_config: CLPConfig, + clp_config: CLPConfig, + container_clp_config: CLPConfig, ): component_name = WEBUI_COMPONENT_NAME logger.info(f"Initializing {component_name}...") container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" client_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "client" / "settings.json" + get_clp_home() / "var" / "www" / "webui" / "client" / "settings.json" ) server_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "server" / "settings.json" + get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "server" / "settings.json" ) validate_webui_config(clp_config, client_settings_json_path, server_settings_json_path) @@ -473,6 +478,7 @@ def start_webui( # FIXME: Handle S3 + def start_garbage_collector( clp_config: CLPConfig, ): @@ -560,20 +566,12 @@ def main(argv): # FIXME: handle this num_workers = multiprocessing.cpu_count() // 2 - container_clp_config, mounts = generate_container_config(clp_config, clp_home) + container_clp_config = generate_docker_compose_container_config(clp_config) # Create necessary directories clp_config.data_directory.mkdir(parents=True, exist_ok=True) clp_config.logs_directory.mkdir(parents=True, exist_ok=True) - # Set container services' hosts in container config - container_clp_config.database.host = DB_COMPONENT_NAME - container_clp_config.queue.host = QUEUE_COMPONENT_NAME - container_clp_config.redis.host = REDIS_COMPONENT_NAME - container_clp_config.results_cache.host = RESULTS_CACHE_COMPONENT_NAME - container_clp_config.query_scheduler.host = QUERY_SCHEDULER_COMPONENT_NAME - container_clp_config.reducer.host = REDUCER_COMPONENT_NAME - dump_shared_container_config(container_clp_config, clp_config) env_dict["CLP_USER_ID"] = os.getuid() @@ -585,20 +583,33 @@ def main(argv): env_dict["CLP_HOST_ARCHIVE_OUTPUT_DIR"] = str(clp_config.archive_output.get_directory()) env_dict["CLP_HOST_STREAM_OUTPUT_DIR"] = str(clp_config.stream_output.get_directory()) + if clp_config.aws_config_directory is not None: + env_dict["CLP_HOST_AWS_CONFIG_DIR"] = str(clp_config.aws_config_directory) + if StorageType.S3 == clp_config.archive_output.storage.type: + clp_config.archive_output.storage.staging_directory.mkdir(parents=True, exist_ok=True) + env_dict["CLP_HOST_ARCHIVE_STAGING_DIR"] = str( + clp_config.archive_output.storage.staging_directory + ) + if StorageType.S3 == clp_config.stream_output.storage.type: + clp_config.stream_output.storage.staging_directory.mkdir(parents=True, exist_ok=True) + env_dict["CLP_HOST_STREAM_STAGING_DIR"] = str( + clp_config.stream_output.storage.staging_directory + ) + try: conf_dir = clp_home / "etc" # Start components - start_db( clp_config, conf_dir) - start_queue( clp_config) - start_redis( clp_config, conf_dir) - start_results_cache( clp_config, conf_dir) + start_db(clp_config, conf_dir) + start_queue(clp_config) + start_redis(clp_config, conf_dir) + start_results_cache(clp_config, conf_dir) start_compression_scheduler(clp_config) start_query_scheduler(clp_config) start_compression_worker(clp_config, num_workers) - start_query_worker( clp_config, num_workers) - start_reducer( clp_config, num_workers) - start_webui( clp_config, container_clp_config) + start_query_worker(clp_config, num_workers) + start_reducer(clp_config, num_workers) + start_webui(clp_config, container_clp_config) start_garbage_collector(clp_config) with open(f"{clp_home}/.env", "w") as env_file: diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 175ab38f7d..3d9932a2c5 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -793,7 +793,7 @@ class CLPConfig(BaseModel): database: Database = Database() queue: Queue = Queue() redis: Redis = Redis() - reducer: Reducer() = Reducer() + reducer: Reducer = Reducer() results_cache: ResultsCache = ResultsCache() compression_scheduler: CompressionScheduler = CompressionScheduler() query_scheduler: QueryScheduler = QueryScheduler() @@ -932,15 +932,15 @@ def get_runnable_components(self) -> Set[str]: return ALL_COMPONENTS def dump_to_primitive_dict(self): - custom_serialized_fields = ( + custom_serialized_fields = { "database", "queue", "redis", "logs_input", "archive_output", "stream_output", - ) - d = self.dict(exclude=set(custom_serialized_fields)) + } + d = self.dict(exclude=custom_serialized_fields) for key in custom_serialized_fields: d[key] = getattr(self, key).dump_to_primitive_dict() diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 03410db3f2..0cd1c84aa1 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -1,13 +1,14 @@ +# FIXME: avoid hardcoding data / log paths secrets: CLP_DB_PASS_FILE: environment: "CLP_DB_PASS" services: db: - image: ${CLP_DB_IMAGE:-mysql:8.0.23} + image: "${CLP_DB_IMAGE:-mysql:8.0.23}" container_name: "database" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" secrets: - - CLP_DB_PASS_FILE + - "CLP_DB_PASS_FILE" environment: MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" MYSQL_USER: "${CLP_DB_USER}" @@ -15,8 +16,8 @@ services: MYSQL_DATABASE: "${CLP_DB_NAME}" volumes: - "./etc/mysql/conf.d/logging.cnf:/etc/mysql/conf.d/logging.cnf:ro" - - "./var/data/database:/var/lib/mysql" - - "./var/log/database:/var/log/mysql" + - "${CLP_HOST_DATA_DIR:-./var/data}/database:/var/lib/mysql" + - "${CLP_HOST_LOGS_DIR:-./var/log}/database:/var/log/mysql" networks: - "clp-network" ports: @@ -31,7 +32,7 @@ services: "--password=${CLP_DB_PASS}" ] start_period: "10s" - start_interval: "2s" + start_interval: "1s" interval: "30s" timeout: "10s" retries: 3 @@ -52,7 +53,7 @@ services: "--storage-engine", "${CLP_STORAGE_ENGINE}" ] volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml" networks: - "clp-network" depends_on: @@ -68,7 +69,7 @@ services: RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" volumes: - - "./var/log/queue:/var/log/rabbitmq" + - "${CLP_HOST_LOGS_DIR:-./var/log}/queue:/var/log/rabbitmq" networks: - "clp-network" ports: @@ -79,7 +80,7 @@ services: "rabbitmq-diagnostics", "check_running" ] start_period: "10s" - start_interval: "2s" + start_interval: "1s" interval: "30s" timeout: "10s" retries: 3 @@ -95,8 +96,8 @@ services: ] volumes: - "./etc/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro" - - "./var/log/redis:/var/log/redis" - - "./var/data/redis:/data" + - "${CLP_HOST_LOGS_DIR:-./var/log}/redis:/var/log/redis" + - "${CLP_HOST_DATA_DIR:-./var/data}/redis:/data" networks: - "clp-network" ports: @@ -111,9 +112,9 @@ services: "PING" ] start_period: "10s" - start_interval: "2s" - interval: 30s - timeout: 10s + start_interval: "1s" + interval: "30s" + timeout: "10s" retries: 3 results-cache: @@ -126,18 +127,18 @@ services: ] volumes: - "./etc/mongo:/etc/mongo:ro" - - "./var/data/results_cache:/data/db" - - "./var/log/results_cache:/var/log/mongodb" + - "${CLP_HOST_DATA_DIR:-./var/data}/results_cache:/data/db" + - "${CLP_HOST_LOGS_DIR:-./var/log}/results_cache:/var/log/mongodb" networks: - "clp-network" ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" healthcheck: - test: echo 'db.runCommand("ping").ok' | mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet + test: "echo 'db.runCommand(\"ping\").ok' | mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet" start_period: "10s" - start_interval: "2s" - interval: 30s - timeout: 10s + start_interval: "1s" + interval: "30s" + timeout: "10s" retries: 3 results-cache-indices-creator: @@ -172,10 +173,11 @@ services: CLP_DB_USER: "${CLP_DB_USER}" CLP_DB_PASS: "${CLP_DB_PASS}" volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" - - "./var/log:/var/log" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" # FIXME: why not call this /mnt/input? - "/:/mnt/logs:ro" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" networks: - "clp-network" depends_on: @@ -191,12 +193,6 @@ services: "-m", "job_orchestration.scheduler.compress.compression_scheduler", "--config", "/etc/clp-config.yml" ] - healthcheck: - # FIXME - test: [ - "CMD", - "true" - ] query-scheduler: image: "clp-package-x86-ubuntu-jammy:dev" @@ -211,8 +207,8 @@ services: CLP_DB_USER: "${CLP_DB_USER}" CLP_DB_PASS: "${CLP_DB_PASS}" volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" - - "./var/log:/var/log" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" networks: - "clp-network" depends_on: @@ -229,11 +225,18 @@ services: "--config", "/etc/clp-config.yml" ] healthcheck: - # FIXME + # FIXME: need to suppressing warnings in the schduler for reading 0 out of 8 expected bytes test: [ "CMD", - "true" + "bash", + "-c", + "< /dev/tcp/query_scheduler/7000" ] + start_period: "10s" + start_interval: "1s" + interval: "30s" + timeout: "10s" + retries: 3 compression-worker: image: "clp-package-x86-ubuntu-jammy:dev" @@ -249,19 +252,17 @@ services: CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" CLP_CONFIG_PATH: "/etc/clp-config.yml" volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" - - "./var/log/compression_worker:/var/log/compression_worker" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/compression_worker:/var/log/compression_worker" # FIXME: why dump "var/data/compression-job-4-task-4-db-config.yml"? - - "./var/data:/var/data" - - "./var/data/archives:/var/data/archives" + - "${CLP_HOST_DATA_DIR:-./var/data}:/var/data" + - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" # FIXME: why not call this /mnt/input? - "/:/mnt/logs:ro" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" + - "${CLP_HOST_ARCHIVE_STAGING_DIR:-./var/data/staged-archives}:/var/data/staged-archives" networks: - "clp-network" - depends_on: - compression-scheduler: - condition: "service_healthy" - # FIXME: add healthcheck command: [ "python3", "-u", @@ -289,15 +290,14 @@ services: CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" CLP_CONFIG_PATH: "/etc/clp-config.yml" volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" - - "./var/log/query_worker:/var/log/query_worker" - - "./var/data/archives:/var/data/archives" - - "./var/data/streams:/var/data/streams" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/query_worker:/var/log/query_worker" + - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" + - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" + - "${CLP_HOST_STREAM_STAGING_DIR:-./var/data/staged-streams}:/var/data/staged-streams" networks: - "clp-network" - depends_on: - query-scheduler: - condition: "service_healthy" command: [ "python3", "-u", @@ -321,8 +321,8 @@ services: CLP_LOGS_DIR: "/var/log/reducer" CLP_LOGGING_LEVEL: "${CLP_REDUCER_LOGGING_LEVEL:-INFO}" volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" - - "./var/log/reducer:/var/log/reducer" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/reducer:/var/log/reducer" networks: - "clp-network" depends_on: @@ -350,11 +350,10 @@ services: CLP_DB_PASS: "${CLP_DB_PASS}" NODE_ENV: "production" volumes: - - "./var/data/streams:/var/data/streams" - - type: "bind" - source: "./var/www/webui/server/dist/server/settings.json" - target: "/opt/clp/var/www/webui/server/dist/server/settings.json" - read_only: true + - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" + - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" + - "./var/www/webui/server/dist/server/settings.json:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" networks: - "clp-network" ports: @@ -368,6 +367,13 @@ services: condition: "service_completed_successfully" results-cache-indices-creator: condition: "service_completed_successfully" + healthcheck: + test: [ + "CMD", + "bash", + "-c", + "< /dev/tcp/webui/4000" + ] garbage-collector: image: "clp-package-x86-ubuntu-jammy:dev" @@ -381,7 +387,7 @@ services: CLP_DB_USER: "${CLP_DB_USER}" CLP_DB_PASS: "${CLP_DB_PASS}" volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_GC_LOGS_DIR}:/var/log/garbage_collector" networks: - "clp-network" @@ -400,4 +406,4 @@ services: networks: clp-network: - driver: bridge \ No newline at end of file + driver: "bridge" From c229fe035fc762ce29246d145c18ed7ef674dbd1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 08:56:22 -0400 Subject: [PATCH 004/238] lint --- tools/deployment/package/docker-compose.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 0cd1c84aa1..5f445f0b45 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -134,7 +134,9 @@ services: ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" healthcheck: - test: "echo 'db.runCommand(\"ping\").ok' | mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet" + test: >- + echo 'db.runCommand("ping").ok' | + mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet start_period: "10s" start_interval: "1s" interval: "30s" @@ -167,7 +169,8 @@ services: environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1}" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} CLP_LOGS_DIR: "/var/log" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_DB_USER: "${CLP_DB_USER}" @@ -201,7 +204,8 @@ services: environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0}" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} CLP_LOGS_DIR: "/var/log" CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_DB_USER: "${CLP_DB_USER}" @@ -245,7 +249,8 @@ services: environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1}" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} CLP_HOME: "/opt/clp" CLP_LOGS_DIR: "/var/log/compression_worker" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" @@ -283,7 +288,8 @@ services: environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0}" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} CLP_HOME: "/opt/clp" CLP_LOGS_DIR: "/var/log/query_worker" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" @@ -352,7 +358,8 @@ services: volumes: - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - - "./var/www/webui/server/dist/server/settings.json:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" + - "./var/www/webui/server/dist/server/settings.json\ +:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" networks: - "clp-network" From a850f6948a01235c5f1c1610ce2eab183f9f241d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 09:02:14 -0400 Subject: [PATCH 005/238] add Docker Compose launch --- .../clp_package_utils/scripts/start_clp.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index c163df9e0a..b3954c34f7 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -622,16 +622,17 @@ def main(argv): logger.exception("Failed to initialize CLP.") return -1 - # try: - # subprocess.run( - # "docker compose up", - # shell=True, - # stderr=subprocess.STDOUT, - # check=True, - # ) - # except subprocess.CalledProcessError: - # logger.exception("Failed to start CLP.") - # return -1 + try: + logger.info(f"Starting CLP with Docker Compose...") + subprocess.run( + "docker compose up -d", + shell=True, + stderr=subprocess.STDOUT, + check=True, + ) + except subprocess.CalledProcessError: + logger.exception("Failed to start CLP.") + return -1 return 0 From b28323139c292286ac75330c98173556a9391edd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 09:10:22 -0400 Subject: [PATCH 006/238] remove unused import --- .../clp_package_utils/scripts/start_clp.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index b3954c34f7..4519cb1310 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -8,11 +8,9 @@ import subprocess import sys import time -import uuid from typing import Any, Dict, List, Optional from clp_py_utils.clp_config import ( - ALL_TARGET_NAME, AwsAuthType, CLPConfig, COMPRESSION_JOBS_TABLE_NAME, @@ -21,7 +19,6 @@ CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME, GARBAGE_COLLECTOR_COMPONENT_NAME, - get_components_for_target, QUERY_JOBS_TABLE_NAME, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, @@ -38,38 +35,26 @@ get_datasets_table_name, get_files_table_name, ) -from clp_py_utils.s3_utils import generate_container_auth_options -from job_orchestration.scheduler.constants import QueueName -from pydantic import BaseModel from clp_package_utils.general import ( check_dependencies, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, - CLPDockerMounts, CONTAINER_CLP_HOME, DockerMount, - DockerMountType, dump_shared_container_config, - generate_container_config, generate_docker_compose_container_config, - get_celery_connection_env_vars_list, get_clp_home, - get_common_env_vars_list, - get_credential_env_vars_list, is_container_exited, is_container_running, - is_retention_period_configured, load_config_file, validate_and_load_db_credentials_file, validate_and_load_queue_credentials_file, validate_and_load_redis_credentials_file, validate_db_config, - validate_log_directory, validate_logs_input_config, validate_output_storage_config, validate_queue_config, validate_redis_config, - validate_reducer_config, validate_results_cache_config, validate_retention_config, validate_webui_config, From 743268cfc59d7afc4e7ed7e83bf7c04522aa76bc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 09:23:18 -0400 Subject: [PATCH 007/238] add proper stop support; reduce stop_grace_period from 10s to 3s --- .../clp_package_utils/scripts/start_clp.py | 5 +- .../clp_package_utils/scripts/stop_clp.py | 94 ++----------------- tools/deployment/package/docker-compose.yml | 17 +++- 3 files changed, 27 insertions(+), 89 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 4519cb1310..2f44d064c3 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -607,11 +607,10 @@ def main(argv): logger.exception("Failed to initialize CLP.") return -1 + logger.info(f"Starting CLP using Docker Compose...") try: - logger.info(f"Starting CLP with Docker Compose...") subprocess.run( - "docker compose up -d", - shell=True, + ["docker", "compose", "up", "-d"], stderr=subprocess.STDOUT, check=True, ) diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 899b7a96a3..632bc8f7f1 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -120,92 +120,16 @@ def main(argv): logger.exception("Failed to load config.") return -1 + logger.info("Stopping all CLP containers using Docker Compose...") try: - # Read instance ID from file - logs_dir = clp_config.logs_directory - instance_id_file_path = logs_dir / "instance-id" - if not (logs_dir.exists() and logs_dir.is_dir() and instance_id_file_path.exists()): - # No instance ID file, so nothing to do - return 0 - with open(instance_id_file_path, "r") as f: - instance_id = f.readline() - - already_exited_containers = [] - force = parsed_args.force - if target in (ALL_TARGET_NAME, GARBAGE_COLLECTOR_COMPONENT_NAME): - container_name = f"clp-{GARBAGE_COLLECTOR_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - if target in (ALL_TARGET_NAME, WEBUI_COMPONENT_NAME): - container_name = f"clp-{WEBUI_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - if target in (ALL_TARGET_NAME, REDUCER_COMPONENT_NAME): - container_name = f"clp-{REDUCER_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - - container_config_file_path = logs_dir / f"{container_name}.yml" - if container_config_file_path.exists(): - container_config_file_path.unlink() - if target in (ALL_TARGET_NAME, QUERY_WORKER_COMPONENT_NAME): - container_name = f"clp-{QUERY_WORKER_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - if target in (ALL_TARGET_NAME, COMPRESSION_WORKER_COMPONENT_NAME): - container_name = f"clp-{COMPRESSION_WORKER_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - if target in (ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, QUERY_SCHEDULER_COMPONENT_NAME): - container_name = f"clp-{QUERY_SCHEDULER_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - - container_config_file_path = logs_dir / f"{container_name}.yml" - if container_config_file_path.exists(): - container_config_file_path.unlink() - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - ): - container_name = f"clp-{COMPRESSION_SCHEDULER_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - - container_config_file_path = logs_dir / f"{container_name}.yml" - if container_config_file_path.exists(): - container_config_file_path.unlink() - if target in (ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, REDIS_COMPONENT_NAME): - container_name = f"clp-{REDIS_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - - redis_config_file_path = logs_dir / f"{container_name}.conf" - if redis_config_file_path.exists(): - redis_config_file_path.unlink() - if target in (ALL_TARGET_NAME, RESULTS_CACHE_COMPONENT_NAME): - container_name = f"clp-{RESULTS_CACHE_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - if target in (ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, QUEUE_COMPONENT_NAME): - container_name = f"clp-{QUEUE_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - - queue_config_file_path = logs_dir / f"{container_name}.conf" - if queue_config_file_path.exists(): - queue_config_file_path.unlink() - if target in (ALL_TARGET_NAME, DB_COMPONENT_NAME): - container_name = f"clp-{DB_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - - if already_exited_containers: - container_list = " ".join(already_exited_containers) - logger.warning( - f"The following containers have already exited and were not removed:" - f" {container_list}" - ) - logger.warning(f"Run with --force to remove them") - elif target in ALL_TARGET_NAME: - # NOTE: We can only remove the instance ID file if all containers have been stopped. - # Currently, we only remove the instance file when all containers are stopped at once. - # If a single container is stopped, it's expensive to check if the others are running, - # so instead we don't remove the instance file. In the worst case, a user will have to - # remove it manually. - instance_id_file_path.unlink() - except: - logger.exception("Failed to stop CLP.") + subprocess.run( + ["docker", "compose", "down"], + stderr=subprocess.STDOUT, + check=True, + ) + logger.info("All CLP containers stopped.") + except subprocess.CalledProcessError: + logger.exception("Failed to stop CLP containers using Docker Compose.") return -1 return 0 diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 5f445f0b45..22f4de3d28 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -1,9 +1,12 @@ -# FIXME: avoid hardcoding data / log paths +x-service-defaults: &service_defaults + stop_grace_period: "3s" + secrets: CLP_DB_PASS_FILE: environment: "CLP_DB_PASS" services: db: + <<: *service_defaults image: "${CLP_DB_IMAGE:-mysql:8.0.23}" container_name: "database" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -38,6 +41,7 @@ services: retries: 3 db-table-creator: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "db_table_creator" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -61,6 +65,7 @@ services: condition: "service_healthy" queue: + <<: *service_defaults image: "rabbitmq:3.9.8" container_name: "queue" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -86,6 +91,7 @@ services: retries: 3 redis: + <<: *service_defaults image: "redis:7.2.4" container_name: "redis" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -118,6 +124,7 @@ services: retries: 3 results-cache: + <<: *service_defaults image: "mongo:7.0.1" container_name: "results_cache" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -144,6 +151,7 @@ services: retries: 3 results-cache-indices-creator: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "results_cache_indices_creator" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -163,6 +171,7 @@ services: condition: "service_healthy" compression-scheduler: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "compression_scheduler" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -198,6 +207,7 @@ services: ] query-scheduler: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "query_scheduler" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -243,6 +253,7 @@ services: retries: 3 compression-worker: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "compression_worker" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -282,6 +293,7 @@ services: ] query-worker: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "query_worker" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -318,6 +330,7 @@ services: ] reducer: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "reducer" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -345,6 +358,7 @@ services: ] webui: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "webui" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -383,6 +397,7 @@ services: ] garbage-collector: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "garbage_collector" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" From ee1b9f28f23ced8fb3fc65e72e371530ca0b4df5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 09:33:32 -0400 Subject: [PATCH 008/238] de-duplicate docker-compose.yml --- tools/deployment/package/docker-compose.yml | 88 +++++---------------- 1 file changed, 20 insertions(+), 68 deletions(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 22f4de3d28..9f79b41aef 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -1,15 +1,28 @@ x-service-defaults: &service_defaults + networks: ["clp-network"] stop_grace_period: "3s" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + +x-healthcheck-defaults: &healthcheck_defaults + start_period: "10s" + start_interval: "1s" + interval: "30s" + timeout: "10s" + retries: 3 + +networks: + clp-network: + driver: "bridge" secrets: CLP_DB_PASS_FILE: environment: "CLP_DB_PASS" + services: db: <<: *service_defaults image: "${CLP_DB_IMAGE:-mysql:8.0.23}" container_name: "database" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" secrets: - "CLP_DB_PASS_FILE" environment: @@ -21,11 +34,10 @@ services: - "./etc/mysql/conf.d/logging.cnf:/etc/mysql/conf.d/logging.cnf:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/database:/var/lib/mysql" - "${CLP_HOST_LOGS_DIR:-./var/log}/database:/var/log/mysql" - networks: - - "clp-network" ports: - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" healthcheck: + <<: *healthcheck_defaults test: [ "CMD", "mysqladmin", "ping", @@ -34,17 +46,11 @@ services: "-u", "${CLP_DB_USER}", "--password=${CLP_DB_PASS}" ] - start_period: "10s" - start_interval: "1s" - interval: "30s" - timeout: "10s" - retries: 3 db-table-creator: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "db_table_creator" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" CLP_DB_USER: "${CLP_DB_USER}" @@ -58,8 +64,6 @@ services: ] volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml" - networks: - - "clp-network" depends_on: db: condition: "service_healthy" @@ -68,33 +72,25 @@ services: <<: *service_defaults image: "rabbitmq:3.9.8" container_name: "queue" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/queue:/var/log/rabbitmq" - networks: - - "clp-network" ports: - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" healthcheck: + <<: *healthcheck_defaults test: [ "CMD", "rabbitmq-diagnostics", "check_running" ] - start_period: "10s" - start_interval: "1s" - interval: "30s" - timeout: "10s" - retries: 3 redis: <<: *service_defaults image: "redis:7.2.4" container_name: "redis" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" command: [ "redis-server", "/usr/local/etc/redis/redis.conf", @@ -104,11 +100,10 @@ services: - "./etc/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}/redis:/var/log/redis" - "${CLP_HOST_DATA_DIR:-./var/data}/redis:/data" - networks: - - "clp-network" ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" healthcheck: + <<: *healthcheck_defaults test: [ "CMD", "redis-cli", @@ -117,17 +112,11 @@ services: "-a", "${CLP_REDIS_PASS}", "PING" ] - start_period: "10s" - start_interval: "1s" - interval: "30s" - timeout: "10s" - retries: 3 results-cache: <<: *service_defaults image: "mongo:7.0.1" container_name: "results_cache" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" command: [ "--config", "/etc/mongo/mongod.conf", "--bind_ip", "0.0.0.0", @@ -136,25 +125,18 @@ services: - "./etc/mongo:/etc/mongo:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/results_cache:/data/db" - "${CLP_HOST_LOGS_DIR:-./var/log}/results_cache:/var/log/mongodb" - networks: - - "clp-network" ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" healthcheck: + <<: *healthcheck_defaults test: >- echo 'db.runCommand("ping").ok' | mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet - start_period: "10s" - start_interval: "1s" - interval: "30s" - timeout: "10s" - retries: 3 results-cache-indices-creator: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "results_cache_indices_creator" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" command: [ @@ -164,8 +146,6 @@ services: "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", ] - networks: - - "clp-network" depends_on: results-cache: condition: "service_healthy" @@ -174,7 +154,6 @@ services: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "compression_scheduler" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" @@ -190,8 +169,6 @@ services: # FIXME: why not call this /mnt/input? - "/:/mnt/logs:ro" - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - networks: - - "clp-network" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -210,7 +187,6 @@ services: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "query_scheduler" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" @@ -223,8 +199,6 @@ services: volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" - networks: - - "clp-network" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -239,6 +213,7 @@ services: "--config", "/etc/clp-config.yml" ] healthcheck: + <<: *healthcheck_defaults # FIXME: need to suppressing warnings in the schduler for reading 0 out of 8 expected bytes test: [ "CMD", @@ -246,17 +221,11 @@ services: "-c", "< /dev/tcp/query_scheduler/7000" ] - start_period: "10s" - start_interval: "1s" - interval: "30s" - timeout: "10s" - retries: 3 compression-worker: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "compression_worker" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" @@ -277,8 +246,6 @@ services: - "/:/mnt/logs:ro" - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_ARCHIVE_STAGING_DIR:-./var/data/staged-archives}:/var/data/staged-archives" - networks: - - "clp-network" command: [ "python3", "-u", @@ -296,7 +263,6 @@ services: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "query_worker" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" @@ -314,8 +280,6 @@ services: - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_STREAM_STAGING_DIR:-./var/data/staged-streams}:/var/data/staged-streams" - networks: - - "clp-network" command: [ "python3", "-u", @@ -333,7 +297,6 @@ services: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "reducer" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" CLP_HOME: "/opt/clp" @@ -342,8 +305,6 @@ services: volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}/reducer:/var/log/reducer" - networks: - - "clp-network" depends_on: query-scheduler: condition: "service_healthy" @@ -361,7 +322,6 @@ services: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "webui" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" HOST: "0.0.0.0" @@ -375,8 +335,6 @@ services: - "./var/www/webui/server/dist/server/settings.json\ :/opt/clp/var/www/webui/server/dist/server/settings.json:ro" - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - networks: - - "clp-network" ports: - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" command: [ @@ -389,6 +347,7 @@ services: results-cache-indices-creator: condition: "service_completed_successfully" healthcheck: + <<: *healthcheck_defaults test: [ "CMD", "bash", @@ -400,7 +359,6 @@ services: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "garbage_collector" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" CLP_HOME: "/opt/clp" @@ -411,8 +369,6 @@ services: volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_GC_LOGS_DIR}:/var/log/garbage_collector" - networks: - - "clp-network" depends_on: query-scheduler: condition: "service_healthy" @@ -425,7 +381,3 @@ services: "-m", "job_orchestration.garbage_collector.garbage_collector", "--config", "/etc/clp-config.yml", ] - -networks: - clp-network: - driver: "bridge" From ec0cfa587c212030bc0cb85e034e89f8c1624f6e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 09:53:58 -0400 Subject: [PATCH 009/238] replace hardcoded image name with environment variable for container configuration --- .../clp_package_utils/scripts/start_clp.py | 2 ++ tools/deployment/package/docker-compose.yml | 18 +++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 2f44d064c3..c8eb0334ce 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -559,6 +559,8 @@ def main(argv): dump_shared_container_config(container_clp_config, clp_config) + env_dict["CLP_PACKAGE_CONTAINER"] = "clp-package-x86-ubuntu-jammy:dev" + env_dict["CLP_USER_ID"] = os.getuid() env_dict["CLP_GROUP_ID"] = os.getgid() env_dict["CLP_STORAGE_ENGINE"] = clp_config.package.storage_engine diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 9f79b41aef..31efd363a9 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -49,7 +49,7 @@ services: db-table-creator: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "db_table_creator" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -135,7 +135,7 @@ services: results-cache-indices-creator: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "results_cache_indices_creator" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -152,7 +152,7 @@ services: compression-scheduler: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "compression_scheduler" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -185,7 +185,7 @@ services: query-scheduler: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "query_scheduler" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -224,7 +224,7 @@ services: compression-worker: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "compression_worker" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -261,7 +261,7 @@ services: query-worker: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "query_worker" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -295,7 +295,7 @@ services: reducer: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "reducer" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -320,7 +320,7 @@ services: webui: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "webui" environment: NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" @@ -357,7 +357,7 @@ services: garbage-collector: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "garbage_collector" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" From ab68c259eb63eb353c69750baa657f7c0cebdd22 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 09:55:48 -0400 Subject: [PATCH 010/238] remove FIXME --- tools/deployment/package/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 31efd363a9..fc39782f0c 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -239,10 +239,8 @@ services: volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}/compression_worker:/var/log/compression_worker" - # FIXME: why dump "var/data/compression-job-4-task-4-db-config.yml"? - "${CLP_HOST_DATA_DIR:-./var/data}:/var/data" - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" - # FIXME: why not call this /mnt/input? - "/:/mnt/logs:ro" - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_ARCHIVE_STAGING_DIR:-./var/data/staged-archives}:/var/data/staged-archives" From 934a83c394a1a8bc478b6163583e2e92aa28994e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 10:10:40 -0400 Subject: [PATCH 011/238] remove FIXME --- .../clp_package_utils/scripts/start_clp.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index c8eb0334ce..307574511e 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -260,7 +260,6 @@ def start_compression_scheduler( clp_config.compression_scheduler.logging_level ) - # FIXME: handle S3 def start_query_scheduler( @@ -275,8 +274,6 @@ def start_query_scheduler( env_dict["CLP_HOST_QUERY_SCHEDULER_LOGS_DIR"] = str(logs_dir) env_dict["CLP_QUERY_SCHEDULER_LOGGING_LEVEL"] = clp_config.query_scheduler.logging_level - # FIXME: handle S3 - def start_compression_worker( clp_config: CLPConfig, @@ -296,8 +293,6 @@ def start_compression_worker( clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) - # FIXME: handle S3 - def start_query_worker( clp_config: CLPConfig, @@ -317,8 +312,6 @@ def start_query_worker( clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) - # FIXME: handle S3 - def start_reducer( clp_config: CLPConfig, @@ -461,8 +454,6 @@ def start_webui( env_dict["CLP_WEBUI_HOST"] = get_ip_from_hostname(clp_config.webui.host) env_dict["CLP_WEBUI_PORT"] = clp_config.webui.port - # FIXME: Handle S3 - def start_garbage_collector( clp_config: CLPConfig, @@ -484,7 +475,6 @@ def add_num_workers_argument(parser): default=multiprocessing.cpu_count(), help="Number of workers to start", ) - # FIXME: handle S3 def main(argv): @@ -548,7 +538,7 @@ def main(argv): logger.exception("Failed to load config.") return -1 - # FIXME: handle this + # TODO: Rely on Docker Compose to spawn multiple workers num_workers = multiprocessing.cpu_count() // 2 container_clp_config = generate_docker_compose_container_config(clp_config) From 83cc9d12ac05b876df3623c723694f2e2aff0a87 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 10:39:50 -0400 Subject: [PATCH 012/238] reformat --- tools/deployment/package/docker-compose.yml | 210 ++++++++++---------- 1 file changed, 104 insertions(+), 106 deletions(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index fc39782f0c..8501be6dae 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -4,11 +4,11 @@ x-service-defaults: &service_defaults user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" x-healthcheck-defaults: &healthcheck_defaults - start_period: "10s" - start_interval: "1s" interval: "30s" - timeout: "10s" retries: 3 + start_interval: "1s" + start_period: "10s" + timeout: "10s" networks: clp-network: @@ -21,21 +21,21 @@ secrets: services: db: <<: *service_defaults - image: "${CLP_DB_IMAGE:-mysql:8.0.23}" container_name: "database" - secrets: - - "CLP_DB_PASS_FILE" + image: "${CLP_DB_IMAGE:-mysql:8.0.23}" environment: + MYSQL_DATABASE: "${CLP_DB_NAME}" + MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" MYSQL_USER: "${CLP_DB_USER}" - MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" - MYSQL_DATABASE: "${CLP_DB_NAME}" + secrets: + - "CLP_DB_PASS_FILE" + ports: + - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" volumes: - - "./etc/mysql/conf.d/logging.cnf:/etc/mysql/conf.d/logging.cnf:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/database:/var/lib/mysql" - "${CLP_HOST_LOGS_DIR:-./var/log}/database:/var/log/mysql" - ports: - - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" + - "./etc/mysql/conf.d/logging.cnf:/etc/mysql/conf.d/logging.cnf:ro" healthcheck: <<: *healthcheck_defaults test: [ @@ -49,12 +49,17 @@ services: db-table-creator: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "db_table_creator" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - CLP_DB_USER: "${CLP_DB_USER}" CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + volumes: + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml" + depends_on: + db: + condition: "service_healthy" command: [ "python3", "-u", @@ -62,24 +67,19 @@ services: "--config", "/etc/clp-config.yml", "--storage-engine", "${CLP_STORAGE_ENGINE}" ] - volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml" - depends_on: - db: - condition: "service_healthy" queue: <<: *service_defaults - image: "rabbitmq:3.9.8" container_name: "queue" + image: "rabbitmq:3.9.8" environment: - RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" - RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" - volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/queue:/var/log/rabbitmq" + RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" + RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" ports: - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" + volumes: + - "${CLP_HOST_LOGS_DIR:-./var/log}/queue:/var/log/rabbitmq" healthcheck: <<: *healthcheck_defaults test: [ @@ -89,19 +89,14 @@ services: redis: <<: *service_defaults - image: "redis:7.2.4" container_name: "redis" - command: [ - "redis-server", - "/usr/local/etc/redis/redis.conf", - "--requirepass", "${CLP_REDIS_PASS}" - ] - volumes: - - "./etc/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/redis:/var/log/redis" - - "${CLP_HOST_DATA_DIR:-./var/data}/redis:/data" + image: "redis:7.2.4" ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" + volumes: + - "${CLP_HOST_DATA_DIR:-./var/data}/redis:/data" + - "${CLP_HOST_LOGS_DIR:-./var/log}/redis:/var/log/redis" + - "./etc/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro" healthcheck: <<: *healthcheck_defaults test: [ @@ -112,33 +107,41 @@ services: "-a", "${CLP_REDIS_PASS}", "PING" ] + command: [ + "redis-server", + "/usr/local/etc/redis/redis.conf", + "--requirepass", "${CLP_REDIS_PASS}" + ] results-cache: <<: *service_defaults - image: "mongo:7.0.1" container_name: "results_cache" - command: [ - "--config", "/etc/mongo/mongod.conf", - "--bind_ip", "0.0.0.0", - ] + image: "mongo:7.0.1" + ports: + - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" volumes: - - "./etc/mongo:/etc/mongo:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/results_cache:/data/db" - "${CLP_HOST_LOGS_DIR:-./var/log}/results_cache:/var/log/mongodb" - ports: - - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" + - "./etc/mongo:/etc/mongo:ro" healthcheck: <<: *healthcheck_defaults test: >- echo 'db.runCommand("ping").ok' | mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet + command: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0", + ] results-cache-indices-creator: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "results_cache_indices_creator" + image: "${CLP_PACKAGE_CONTAINER}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" + depends_on: + results-cache: + condition: "service_healthy" command: [ "python3", "-u", @@ -146,29 +149,25 @@ services: "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", ] - depends_on: - results-cache: - condition: "service_healthy" compression-scheduler: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "compression_scheduler" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} - CLP_LOGS_DIR: "/var/log" - CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" - CLP_DB_USER: "${CLP_DB_USER}" - CLP_DB_PASS: "${CLP_DB_PASS}" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" - # FIXME: why not call this /mnt/input? + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -185,20 +184,20 @@ services: query-scheduler: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "query_scheduler" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} - CLP_LOGS_DIR: "/var/log" - CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" - CLP_DB_USER: "${CLP_DB_USER}" - CLP_DB_PASS: "${CLP_DB_PASS}" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -224,26 +223,26 @@ services: compression-worker: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "compression_worker" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" - CLP_LOGS_DIR: "/var/log/compression_worker" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/compression_worker" CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" - CLP_CONFIG_PATH: "/etc/clp-config.yml" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/compression_worker:/var/log/compression_worker" + - "${CLP_HOST_ARCHIVE_STAGING_DIR:-./var/data/staged-archives}:/var/data/staged-archives" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_DATA_DIR:-./var/data}:/var/data" - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/compression_worker:/var/log/compression_worker" - "/:/mnt/logs:ro" - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - - "${CLP_HOST_ARCHIVE_STAGING_DIR:-./var/data/staged-archives}:/var/data/staged-archives" command: [ "python3", "-u", @@ -259,24 +258,24 @@ services: query-worker: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "query_worker" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} + CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" - CLP_LOGS_DIR: "/var/log/query_worker" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/query_worker" CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" - CLP_CONFIG_PATH: "/etc/clp-config.yml" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/query_worker:/var/log/query_worker" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/query_worker:/var/log/query_worker" - "${CLP_HOST_STREAM_STAGING_DIR:-./var/data/staged-streams}:/var/data/staged-streams" command: [ "python3", @@ -293,13 +292,13 @@ services: reducer: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "reducer" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" CLP_HOME: "/opt/clp" - CLP_LOGS_DIR: "/var/log/reducer" CLP_LOGGING_LEVEL: "${CLP_REDUCER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/reducer" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}/reducer:/var/log/reducer" @@ -318,32 +317,31 @@ services: webui: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "webui" + image: "${CLP_PACKAGE_CONTAINER}" environment: - NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" - HOST: "0.0.0.0" - PORT: "4000" - CLP_DB_USER: "${CLP_DB_USER}" CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + HOST: "0.0.0.0" NODE_ENV: "production" + NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" + PORT: "4000" + ports: + - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" volumes: + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - - "./var/www/webui/server/dist/server/settings.json\ -:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - ports: - - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" - command: [ - "/opt/clp/bin/node-22", - "/opt/clp/var/www/webui/server/dist/server/src/main.js" - ] + - "./var/www/webui/server/dist/server/settings.json:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" depends_on: db-table-creator: condition: "service_completed_successfully" results-cache-indices-creator: condition: "service_completed_successfully" + command: [ + "/opt/clp/bin/node-22", + "/opt/clp/var/www/webui/server/dist/server/src/main.js" + ] healthcheck: <<: *healthcheck_defaults test: [ @@ -355,23 +353,23 @@ services: garbage-collector: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "garbage_collector" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" CLP_HOME: "/opt/clp" - CLP_LOGS_DIR: "/var/log/garbage_collector" CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" - CLP_DB_USER: "${CLP_DB_USER}" - CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_LOGS_DIR: "/var/log/garbage_collector" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_GC_LOGS_DIR}:/var/log/garbage_collector" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" depends_on: - query-scheduler: - condition: "service_healthy" db-table-creator: condition: "service_completed_successfully" + query-scheduler: + condition: "service_healthy" results-cache-indices-creator: condition: "service_completed_successfully" command: [ From 82abc077193ac67267ff419e4fb406a56eab68a0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 10:41:07 -0400 Subject: [PATCH 013/238] reformat --- tools/deployment/package/docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 8501be6dae..f5057ebe3f 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -332,7 +332,8 @@ services: - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - - "./var/www/webui/server/dist/server/settings.json:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" + - "./var/www/webui/server/dist/server/settings.json\ +:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" depends_on: db-table-creator: condition: "service_completed_successfully" From 0fb229416d1fea146d66c2d6a50fc1728812ac7c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 10:43:32 -0400 Subject: [PATCH 014/238] Update garbage collector logs directory mapping --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 1 - tools/deployment/package/docker-compose.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 307574511e..7eb790ac47 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -464,7 +464,6 @@ def start_garbage_collector( logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - env_dict["CLP_GC_LOGS_DIR"] = str(logs_dir) env_dict["CLP_GC_LOGGING_LEVEL"] = clp_config.garbage_collector.logging_level diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index f5057ebe3f..8648d6492a 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -364,8 +364,8 @@ services: CLP_LOGS_DIR: "/var/log/garbage_collector" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_GC_LOGS_DIR}:/var/log/garbage_collector" - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/garbage_collector:/var/log/garbage_collector" depends_on: db-table-creator: condition: "service_completed_successfully" From c8ffb949b4e854bde8203cbfa64e0c8e6961ec0e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 10:46:08 -0400 Subject: [PATCH 015/238] Remove unused component argument parsers --- .../clp_package_utils/scripts/start_clp.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 7eb790ac47..e1e52dbfef 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -488,23 +488,6 @@ def main(argv): help="CLP package configuration file.", ) - component_args_parser = args_parser.add_subparsers(dest="target") - component_args_parser.add_parser(CONTROLLER_TARGET_NAME) - component_args_parser.add_parser(DB_COMPONENT_NAME) - component_args_parser.add_parser(QUEUE_COMPONENT_NAME) - component_args_parser.add_parser(REDIS_COMPONENT_NAME) - component_args_parser.add_parser(RESULTS_CACHE_COMPONENT_NAME) - component_args_parser.add_parser(COMPRESSION_SCHEDULER_COMPONENT_NAME) - component_args_parser.add_parser(QUERY_SCHEDULER_COMPONENT_NAME) - compression_worker_parser = component_args_parser.add_parser(COMPRESSION_WORKER_COMPONENT_NAME) - add_num_workers_argument(compression_worker_parser) - query_worker_parser = component_args_parser.add_parser(QUERY_WORKER_COMPONENT_NAME) - add_num_workers_argument(query_worker_parser) - reducer_server_parser = component_args_parser.add_parser(REDUCER_COMPONENT_NAME) - add_num_workers_argument(reducer_server_parser) - component_args_parser.add_parser(WEBUI_COMPONENT_NAME) - component_args_parser.add_parser(GARBAGE_COLLECTOR_COMPONENT_NAME) - parsed_args = args_parser.parse_args(argv[1:]) try: From 83e902a39ec34ffcb6f57b10c0120478e7903c0a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 10:48:11 -0400 Subject: [PATCH 016/238] Remove unused component argument parsers --- .../clp_package_utils/scripts/start_clp.py | 1 - .../clp_package_utils/scripts/stop_clp.py | 118 +----------------- 2 files changed, 2 insertions(+), 117 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index e1e52dbfef..2d0850eea8 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -261,7 +261,6 @@ def start_compression_scheduler( ) - def start_query_scheduler( clp_config: CLPConfig, ): diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 632bc8f7f1..03981bd474 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -1,125 +1,11 @@ -import argparse import logging -import pathlib import subprocess import sys -from typing import List - -from clp_py_utils.clp_config import ( - ALL_TARGET_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - CONTROLLER_TARGET_NAME, - DB_COMPONENT_NAME, - GARBAGE_COLLECTOR_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - QUEUE_COMPONENT_NAME, - REDIS_COMPONENT_NAME, - REDUCER_COMPONENT_NAME, - RESULTS_CACHE_COMPONENT_NAME, - WEBUI_COMPONENT_NAME, -) - -from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, - get_clp_home, - is_container_exited, - is_container_running, - load_config_file, - validate_and_load_db_credentials_file, - validate_and_load_queue_credentials_file, -) logger = logging.getLogger(__file__) -def stop_running_container(container_name: str, already_exited_containers: List[str], force: bool): - if is_container_running(container_name): - logger.info(f"Stopping {container_name}...") - cmd = ["docker", "stop", container_name] - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Removing {container_name}...") - cmd = ["docker", "rm", container_name] - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Stopped and removed {container_name}.") - elif is_container_exited(container_name): - if force: - logger.info(f"Forcibly removing exited {container_name}...") - cmd = ["docker", "rm", container_name] - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - logger.info(f"Removed {container_name}...") - else: - already_exited_containers.append(container_name) - - -def main(argv): - clp_home = get_clp_home() - default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH - - args_parser = argparse.ArgumentParser(description="Stops CLP") - args_parser.add_argument( - "--config", - "-c", - default=str(default_config_file_path), - help="CLP package configuration file.", - ) - args_parser.add_argument( - "--force", - "-f", - action="store_true", - help="Forcibly remove exited containers", - ) - - component_args_parser = args_parser.add_subparsers(dest="target") - component_args_parser.add_parser(CONTROLLER_TARGET_NAME) - component_args_parser.add_parser(DB_COMPONENT_NAME) - component_args_parser.add_parser(QUEUE_COMPONENT_NAME) - component_args_parser.add_parser(REDIS_COMPONENT_NAME) - component_args_parser.add_parser(REDUCER_COMPONENT_NAME) - component_args_parser.add_parser(RESULTS_CACHE_COMPONENT_NAME) - component_args_parser.add_parser(COMPRESSION_SCHEDULER_COMPONENT_NAME) - component_args_parser.add_parser(QUERY_SCHEDULER_COMPONENT_NAME) - component_args_parser.add_parser(COMPRESSION_WORKER_COMPONENT_NAME) - component_args_parser.add_parser(QUERY_WORKER_COMPONENT_NAME) - component_args_parser.add_parser(WEBUI_COMPONENT_NAME) - component_args_parser.add_parser(GARBAGE_COLLECTOR_COMPONENT_NAME) - - parsed_args = args_parser.parse_args(argv[1:]) - - if parsed_args.target: - target = parsed_args.target - else: - target = ALL_TARGET_NAME - - # Validate and load config file - try: - config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) - - # Validate and load necessary credentials - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - DB_COMPONENT_NAME, - ): - validate_and_load_db_credentials_file(clp_config, clp_home, False) - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - QUEUE_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - ): - validate_and_load_queue_credentials_file(clp_config, clp_home, False) - except: - logger.exception("Failed to load config.") - return -1 - +def main(): logger.info("Stopping all CLP containers using Docker Compose...") try: subprocess.run( @@ -136,4 +22,4 @@ def main(argv): if "__main__" == __name__: - sys.exit(main(sys.argv)) + sys.exit(main()) From 5f2e5cdbc68599eabdb2416c6f5c405b675ce03c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 11:22:24 -0400 Subject: [PATCH 017/238] Refactor dependency checks to include docker-compose status validation --- .../clp_package_utils/general.py | 65 +++++-------------- .../clp_package_utils/scripts/start_clp.py | 19 +----- .../clp_package_utils/scripts/stop_clp.py | 8 +++ 3 files changed, 25 insertions(+), 67 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index b27c62606c..963b058157 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -135,65 +135,32 @@ def generate_container_name(job_type: str) -> str: return f"clp-{job_type}-{str(uuid.uuid4())[-4:]}" -def check_dependencies(): +def is_compose_running(): + cmd = ["docker", "compose", "ps", "--quiet"] + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + return bool(output.strip()) + except subprocess.CalledProcessError: + raise EnvironmentError("docker-compose is not installed or not functioning properly.") + + +def check_dependencies(should_compose_run: bool = False): try: subprocess.run( "command -v docker", shell=True, - stdout=subprocess.PIPE, + stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, check=True, ) except subprocess.CalledProcessError: raise EnvironmentError("docker is not installed or available on the path") - try: - subprocess.run( - ["docker", "ps"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True - ) - except subprocess.CalledProcessError: - raise EnvironmentError("docker cannot run without superuser privileges (sudo).") - try: - subprocess.run( - ["docker", "compose", "version"], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - check=True, - ) - except subprocess.CalledProcessError: - raise EnvironmentError("docker-compose is not installed") - -def is_container_running(container_name): - # fmt: off - cmd = [ - "docker", "ps", - # Only return container IDs - "--quiet", - "--filter", f"name={container_name}" - ] - # fmt: on - proc = subprocess.run(cmd, stdout=subprocess.PIPE) - if proc.stdout.decode("utf-8"): - return True - - return False - - -def is_container_exited(container_name): - # fmt: off - cmd = [ - "docker", "ps", - # Only return container IDs - "--quiet", - "--filter", f"name={container_name}", - "--filter", "status=exited" - ] - # fmt: on - proc = subprocess.run(cmd, stdout=subprocess.PIPE) - if proc.stdout.decode("utf-8"): - return True - - return False + is_running = is_compose_running() + if should_compose_run and not is_running: + raise EnvironmentError("docker-compose is not running.") + if not should_compose_run and is_running: + raise EnvironmentError("docker-compose is already running.") def validate_log_directory(logs_dir: pathlib.Path, component_name: str) -> None: diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 2d0850eea8..ae1ac14d48 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -44,8 +44,6 @@ dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, - is_container_exited, - is_container_running, load_config_file, validate_and_load_db_credentials_file, validate_and_load_queue_credentials_file, @@ -68,16 +66,6 @@ def get_ip_from_hostname(hostname: str) -> str: return socket.gethostbyname(hostname) -def container_exists(container_name): - if is_container_running(container_name): - logger.info(f"{container_name} already running.") - return True - elif is_container_exited(container_name): - logger.info(f"{container_name} exited but not removed.") - return True - return False - - def append_docker_options( cmd: List[str], mounts: Optional[List[Optional[DockerMount]]] = None, @@ -490,7 +478,7 @@ def main(argv): parsed_args = args_parser.parse_args(argv[1:]) try: - check_dependencies() + check_dependencies(should_compose_run=False) except: logger.exception("Dependency checking failed.") return -1 @@ -507,11 +495,6 @@ def main(argv): validate_output_storage_config(clp_config) validate_retention_config(clp_config) - # validate_and_load_db_credentials_file(clp_config, clp_home, True) - # validate_and_load_queue_credentials_file(clp_config, clp_home, True) - # validate_and_load_redis_credentials_file(clp_config, clp_home, True) - # validate_worker_config(clp_config) - clp_config.validate_data_dir() clp_config.validate_logs_dir() clp_config.validate_aws_config_dir() diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 03981bd474..39912ce87c 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -2,10 +2,18 @@ import subprocess import sys +from clp_package_utils.general import check_dependencies + logger = logging.getLogger(__file__) def main(): + try: + check_dependencies(should_compose_run=True) + except: + logger.exception("Dependency checking failed.") + return -1 + logger.info("Stopping all CLP containers using Docker Compose...") try: subprocess.run( From edfa9c91a7e891aaa351e1e70152121bde5d514c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 11:25:27 -0400 Subject: [PATCH 018/238] Refactor log directory handling to use constant path definitions --- components/clp-package-utils/clp_package_utils/general.py | 5 ++--- components/clp-py-utils/clp_py_utils/clp_config.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 963b058157..b2a3b67c70 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -15,6 +15,7 @@ from clp_py_utils.clp_config import ( CLP_DEFAULT_CREDENTIALS_FILE_PATH, CLP_DEFAULT_DATA_DIRECTORY_PATH, + CLP_DEFAULT_LOG_DIRECTORY_PATH, CLP_SHARED_CONFIG_FILENAME, CLPConfig, DB_COMPONENT_NAME, @@ -297,11 +298,9 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig """ container_clp_config = clp_config.copy(deep=True) - # FIXME: consider removing credentials_file_path - # Set container paths container_clp_config.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH - container_clp_config.logs_directory = pathlib.Path("/") / "var" / "log" + container_clp_config.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH if StorageType.FS == clp_config.logs_input.type: container_clp_config.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 3d9932a2c5..90fb524f28 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -80,11 +80,11 @@ CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path("etc") / "credentials.yml" CLP_DEFAULT_DATA_DIRECTORY_PATH = pathlib.Path("var") / "data" +CLP_DEFAULT_LOG_DIRECTORY_PATH = pathlib.Path("var") / "log" CLP_DEFAULT_DATASET_NAME = "default" CLP_METADATA_TABLE_PREFIX = "clp_" CLP_SHARED_CONFIG_FILENAME = ".clp-config.yml" - # Environment variable names CLP_DB_USER_ENV_VAR_NAME = "CLP_DB_USER" CLP_DB_PASS_ENV_VAR_NAME = "CLP_DB_PASS" @@ -805,8 +805,8 @@ class CLPConfig(BaseModel): archive_output: ArchiveOutput = ArchiveOutput() stream_output: StreamOutput = StreamOutput() - data_directory: pathlib.Path = pathlib.Path("var") / "data" - logs_directory: pathlib.Path = pathlib.Path("var") / "log" + data_directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH + logs_directory: pathlib.Path = CLP_DEFAULT_LOG_DIRECTORY_PATH aws_config_directory: Optional[pathlib.Path] = None _os_release_file_path: pathlib.Path = PrivateAttr(default=OS_RELEASE_FILE_PATH) From 7b3965e3333f79bf00906a9a30806a74295f18da Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 11:29:05 -0400 Subject: [PATCH 019/238] Add constants for archive and stream directory paths --- .../clp_package_utils/general.py | 16 ++++++++-------- .../clp-py-utils/clp_py_utils/clp_config.py | 12 ++++++++---- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index b2a3b67c70..505379ce26 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -13,9 +13,13 @@ import yaml from clp_py_utils.clp_config import ( + CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH, + CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH, CLP_DEFAULT_CREDENTIALS_FILE_PATH, CLP_DEFAULT_DATA_DIRECTORY_PATH, CLP_DEFAULT_LOG_DIRECTORY_PATH, + CLP_DEFAULT_STREAM_DIRECTORY_PATH, + CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH, CLP_SHARED_CONFIG_FILENAME, CLPConfig, DB_COMPONENT_NAME, @@ -305,21 +309,17 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig container_clp_config.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR if StorageType.FS == clp_config.archive_output.storage.type: - container_clp_config.archive_output.storage.directory = ( - pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" - ) + container_clp_config.archive_output.storage.directory = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH elif StorageType.S3 == clp_config.archive_output.storage.type: container_clp_config.archive_output.storage.staging_directory = ( - CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" + CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH ) if StorageType.FS == clp_config.stream_output.storage.type: - container_clp_config.stream_output.storage.directory = ( - pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" - ) + container_clp_config.stream_output.storage.directory = CLP_DEFAULT_STREAM_DIRECTORY_PATH elif StorageType.S3 == clp_config.stream_output.storage.type: container_clp_config.stream_output.storage.staging_directory = ( - CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" + CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH ) if clp_config.aws_config_directory is not None: diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 90fb524f28..49aa08d4b8 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -80,6 +80,10 @@ CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path("etc") / "credentials.yml" CLP_DEFAULT_DATA_DIRECTORY_PATH = pathlib.Path("var") / "data" +CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" +CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" +CLP_DEFAULT_STREAM_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" +CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" CLP_DEFAULT_LOG_DIRECTORY_PATH = pathlib.Path("var") / "log" CLP_DEFAULT_DATASET_NAME = "default" CLP_METADATA_TABLE_PREFIX = "clp_" @@ -609,19 +613,19 @@ class FsIngestionConfig(FsStorage): class ArchiveFsStorage(FsStorage): - directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" + directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH class StreamFsStorage(FsStorage): - directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" + directory: pathlib.Path = CLP_DEFAULT_STREAM_DIRECTORY_PATH class ArchiveS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" + staging_directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH class StreamS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" + staging_directory: pathlib.Path = CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH def _get_directory_from_storage_config( From cd84be806dccc4bb5efdc0363b2eeb848fd7a32b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 11:33:16 -0400 Subject: [PATCH 020/238] remove unused component groups and functions --- .../clp-py-utils/clp_py_utils/clp_config.py | 56 +------------------ 1 file changed, 1 insertion(+), 55 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 49aa08d4b8..2a0b53cbed 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,7 +1,7 @@ import os import pathlib from enum import auto -from typing import Literal, Optional, Set, Union +from typing import Literal, Optional, Union from dotenv import dotenv_values from pydantic import BaseModel, PrivateAttr, root_validator, validator @@ -29,45 +29,6 @@ WEBUI_COMPONENT_NAME = "webui" GARBAGE_COLLECTOR_COMPONENT_NAME = "garbage_collector" -# Component groups -GENERAL_SCHEDULING_COMPONENTS = { - QUEUE_COMPONENT_NAME, - REDIS_COMPONENT_NAME, -} -COMPRESSION_COMPONENTS = GENERAL_SCHEDULING_COMPONENTS | { - DB_COMPONENT_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, -} -QUERY_COMPONENTS = GENERAL_SCHEDULING_COMPONENTS | { - DB_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - REDUCER_COMPONENT_NAME, -} -UI_COMPONENTS = { - RESULTS_CACHE_COMPONENT_NAME, - WEBUI_COMPONENT_NAME, -} -STORAGE_MANAGEMENT_COMPONENTS = {GARBAGE_COLLECTOR_COMPONENT_NAME} -ALL_COMPONENTS = ( - COMPRESSION_COMPONENTS | QUERY_COMPONENTS | UI_COMPONENTS | STORAGE_MANAGEMENT_COMPONENTS -) - -# Target names -ALL_TARGET_NAME = "" -CONTROLLER_TARGET_NAME = "controller" - -TARGET_TO_COMPONENTS = { - ALL_TARGET_NAME: ALL_COMPONENTS, - CONTROLLER_TARGET_NAME: GENERAL_SCHEDULING_COMPONENTS - | { - COMPRESSION_SCHEDULER_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - } - | STORAGE_MANAGEMENT_COMPONENTS, -} - # Action names ARCHIVE_MANAGER_ACTION_NAME = "archive_manager" @@ -929,12 +890,6 @@ def load_execution_container_name(self): def get_shared_config_file_path(self) -> pathlib.Path: return self.logs_directory / CLP_SHARED_CONFIG_FILENAME - def get_runnable_components(self) -> Set[str]: - if QueryEngine.PRESTO == self.package.query_engine: - return COMPRESSION_COMPONENTS | UI_COMPONENTS - else: - return ALL_COMPONENTS - def dump_to_primitive_dict(self): custom_serialized_fields = { "database", @@ -978,12 +933,3 @@ def dump_to_primitive_dict(self): d["stream_output"] = self.stream_output.dump_to_primitive_dict() return d - - -def get_components_for_target(target: str) -> Set[str]: - if target in TARGET_TO_COMPONENTS: - return TARGET_TO_COMPONENTS[target] - elif target in ALL_COMPONENTS: - return {target} - else: - return set() From aa12bdbc7e929f2a5f61a9ef34f851172ab146be Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 11:36:20 -0400 Subject: [PATCH 021/238] Remove unused CONTROLLER_TARGET_NAME constant from start_clp.py --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index ae1ac14d48..52d1c4318a 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -16,7 +16,6 @@ COMPRESSION_JOBS_TABLE_NAME, COMPRESSION_SCHEDULER_COMPONENT_NAME, COMPRESSION_WORKER_COMPONENT_NAME, - CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME, GARBAGE_COLLECTOR_COMPONENT_NAME, QUERY_JOBS_TABLE_NAME, From 5365722a30b95485ec386039e754e7f0020c71e8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 12:19:09 -0400 Subject: [PATCH 022/238] fix staging dirs --- components/clp-package-utils/clp_package_utils/general.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 505379ce26..aecc7aa89d 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -312,14 +312,14 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig container_clp_config.archive_output.storage.directory = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH elif StorageType.S3 == clp_config.archive_output.storage.type: container_clp_config.archive_output.storage.staging_directory = ( - CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH ) if StorageType.FS == clp_config.stream_output.storage.type: container_clp_config.stream_output.storage.directory = CLP_DEFAULT_STREAM_DIRECTORY_PATH elif StorageType.S3 == clp_config.stream_output.storage.type: container_clp_config.stream_output.storage.staging_directory = ( - CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH ) if clp_config.aws_config_directory is not None: From 21ef70355861132061feccfa172983796118509c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 12:39:36 -0400 Subject: [PATCH 023/238] fix: update command to check if Docker Compose is running --- components/clp-package-utils/clp_package_utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index aecc7aa89d..2f43dbea9b 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -141,7 +141,7 @@ def generate_container_name(job_type: str) -> str: def is_compose_running(): - cmd = ["docker", "compose", "ps", "--quiet"] + cmd = ["docker", "compose", "ls", "--quiet"] try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) return bool(output.strip()) From d2cdfbc02a30c6d104866618035ea42e13d6269b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 12:39:55 -0400 Subject: [PATCH 024/238] add AWS env credentials support --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 4 ++++ tools/deployment/package/docker-compose.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 52d1c4318a..19c235af87 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -523,6 +523,10 @@ def main(argv): env_dict["CLP_HOST_ARCHIVE_OUTPUT_DIR"] = str(clp_config.archive_output.get_directory()) env_dict["CLP_HOST_STREAM_OUTPUT_DIR"] = str(clp_config.stream_output.get_directory()) + # TODO: validate if those are required + env_dict["CLP_AWS_ACCESS_KEY_ID"] = os.getenv("AWS_ACCESS_KEY_ID", "") + env_dict["CLP_AWS_SECRET_ACCESS_KEY"] = os.getenv("AWS_SECRET_ACCESS_KEY", "") + if clp_config.aws_config_directory is not None: env_dict["CLP_HOST_AWS_CONFIG_DIR"] = str(clp_config.aws_config_directory) if StorageType.S3 == clp_config.archive_output.storage.type: diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 8648d6492a..536109daec 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -226,6 +226,8 @@ services: container_name: "compression_worker" image: "${CLP_PACKAGE_CONTAINER}" environment: + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" @@ -261,6 +263,8 @@ services: container_name: "query_worker" image: "${CLP_PACKAGE_CONTAINER}" environment: + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" From f0db07fe00c5fbe623f39b5d4f4a53f81489f8dc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 26 Aug 2025 15:21:16 -0400 Subject: [PATCH 025/238] Update container image name in start_clp.py --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 19c235af87..624b14f9ab 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -512,7 +512,7 @@ def main(argv): dump_shared_container_config(container_clp_config, clp_config) - env_dict["CLP_PACKAGE_CONTAINER"] = "clp-package-x86-ubuntu-jammy:dev" + env_dict["CLP_PACKAGE_CONTAINER"] = "clp-package:dev" env_dict["CLP_USER_ID"] = os.getuid() env_dict["CLP_GROUP_ID"] = os.getgid() From 5f24ce70b3738b5efabb0e5c2fe11a55aafec2d4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 3 Sep 2025 18:34:20 -0400 Subject: [PATCH 026/238] add support for configurable CLP WebUI rate limiting --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 1 + tools/deployment/package/docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 624b14f9ab..7b2e14ccae 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -439,6 +439,7 @@ def start_webui( env_dict["CLP_WEBUI_HOST"] = get_ip_from_hostname(clp_config.webui.host) env_dict["CLP_WEBUI_PORT"] = clp_config.webui.port + env_dict["CLP_WEBUI_RATE_LIMIT"] = clp_config.webui.rate_limit def start_garbage_collector( diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 536109daec..eee49564a1 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -330,6 +330,7 @@ services: NODE_ENV: "production" NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" PORT: "4000" + RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" ports: - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" volumes: From 3df20fce0d50f53f754938d37b471a8b5f82e38a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 3 Sep 2025 18:36:08 -0400 Subject: [PATCH 027/238] update WebUI server path in start_clp.py and docker-compose configuration --- .../clp_package_utils/scripts/start_clp.py | 2 +- tools/deployment/package/docker-compose.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 7b2e14ccae..05991bfd76 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -364,7 +364,7 @@ def start_webui( get_clp_home() / "var" / "www" / "webui" / "client" / "settings.json" ) server_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "server" / "settings.json" + get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "settings.json" ) validate_webui_config(clp_config, client_settings_json_path, server_settings_json_path) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index eee49564a1..d3ea05493f 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -337,8 +337,8 @@ services: - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - - "./var/www/webui/server/dist/server/settings.json\ -:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" + - "./var/www/webui/server/dist/settings.json\ +:/opt/clp/var/www/webui/server/dist/settings.json:ro" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -346,7 +346,7 @@ services: condition: "service_completed_successfully" command: [ "/opt/clp/bin/node-22", - "/opt/clp/var/www/webui/server/dist/server/src/main.js" + "/opt/clp/var/www/webui/server/dist/src/main.js" ] healthcheck: <<: *healthcheck_defaults From c6f81ade93518f4dae1a42f8a75d90c4ae472f7d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 3 Sep 2025 19:21:28 -0400 Subject: [PATCH 028/238] copy docker-compose.yml in package task --- taskfile.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/taskfile.yaml b/taskfile.yaml index 874af69cf0..90a0e886d8 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -113,6 +113,7 @@ tasks: - "components/clp-py-utils/dist/*.whl" - "components/job-orchestration/dist/*.whl" - "components/package-template/src/**/*" + - "tools/deployment/package/docker-compose.yml" generates: ["{{.CHECKSUM_FILE}}"] deps: - "core" @@ -160,6 +161,10 @@ tasks: - |- cd "{{.OUTPUT_DIR}}/var/www/webui" PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm ci --omit=dev + - >- + rsync -a + "tools/deployment/package/docker-compose.yml" + "{{.OUTPUT_DIR}}" # This command must be last - task: "utils:checksum:compute" vars: From db9c20fef293fcac1b776785864c5b4864d4ad8a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 3 Sep 2025 19:48:16 -0400 Subject: [PATCH 029/238] use absolute paths in archive and stream storage configurations --- components/clp-package-utils/clp_package_utils/general.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 2f43dbea9b..a31e00d11c 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -309,14 +309,18 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig container_clp_config.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR if StorageType.FS == clp_config.archive_output.storage.type: - container_clp_config.archive_output.storage.directory = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + container_clp_config.archive_output.storage.directory = ( + pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + ) elif StorageType.S3 == clp_config.archive_output.storage.type: container_clp_config.archive_output.storage.staging_directory = ( pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH ) if StorageType.FS == clp_config.stream_output.storage.type: - container_clp_config.stream_output.storage.directory = CLP_DEFAULT_STREAM_DIRECTORY_PATH + container_clp_config.stream_output.storage.directory = ( + pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH + ) elif StorageType.S3 == clp_config.stream_output.storage.type: container_clp_config.stream_output.storage.staging_directory = ( pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH From ea03e17956cac6fbb178efb79ba6680ffc67bda8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 00:21:41 -0400 Subject: [PATCH 030/238] refactor: centralize environment variable management and enhance validation --- .../clp_package_utils/general.py | 21 +- .../clp_package_utils/scripts/start_clp.py | 276 +++++++----------- .../clp-py-utils/clp_py_utils/clp_config.py | 122 +++++++- tools/deployment/package/docker-compose.yml | 70 ++--- 4 files changed, 271 insertions(+), 218 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index a31e00d11c..302f145e92 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -519,7 +519,13 @@ def validate_and_load_redis_credentials_file( clp_config.redis.load_credentials_from_file(clp_config.credentials_file_path) -def validate_db_config(clp_config: CLPConfig, data_dir: pathlib.Path, logs_dir: pathlib.Path): +def validate_db_config( + clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path +): + if not base_config.exists(): + raise ValueError( + f"{DB_COMPONENT_NAME} base configuration at {str(base_config)} is missing." + ) _validate_data_directory(data_dir, DB_COMPONENT_NAME) validate_log_directory(logs_dir, DB_COMPONENT_NAME) @@ -533,15 +539,14 @@ def validate_queue_config(clp_config: CLPConfig, logs_dir: pathlib.Path): def validate_redis_config( - clp_config: CLPConfig, data_dir: pathlib.Path, logs_dir: pathlib.Path, base_config: pathlib.Path + clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path ): - _validate_data_directory(data_dir, REDIS_COMPONENT_NAME) - validate_log_directory(logs_dir, REDIS_COMPONENT_NAME) - if not base_config.exists(): raise ValueError( f"{REDIS_COMPONENT_NAME} base configuration at {str(base_config)} is missing." ) + _validate_data_directory(data_dir, REDIS_COMPONENT_NAME) + validate_log_directory(logs_dir, REDIS_COMPONENT_NAME) validate_port(f"{REDIS_COMPONENT_NAME}.port", clp_config.redis.host, clp_config.redis.port) @@ -558,8 +563,12 @@ def validate_reducer_config(clp_config: CLPConfig, logs_dir: pathlib.Path, num_w def validate_results_cache_config( - clp_config: CLPConfig, data_dir: pathlib.Path, logs_dir: pathlib.Path + clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path ): + if not base_config.exists(): + raise ValueError( + f"{RESULTS_CACHE_COMPONENT_NAME} base configuration at {str(base_config)} is missing." + ) _validate_data_directory(data_dir, RESULTS_CACHE_COMPONENT_NAME) validate_log_directory(logs_dir, RESULTS_CACHE_COMPONENT_NAME) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 05991bfd76..8cf3b63bc7 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -2,9 +2,10 @@ import json import logging import multiprocessing -import os import pathlib +import shlex import socket +import stat import subprocess import sys import time @@ -57,12 +58,9 @@ validate_webui_config, ) -logger = logging.getLogger(__file__) -env_dict = {} - +LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH -def get_ip_from_hostname(hostname: str) -> str: - return socket.gethostbyname(hostname) +logger = logging.getLogger(__file__) def append_docker_options( @@ -114,7 +112,7 @@ def chown_recursively( subprocess.run(chown_cmd, stdout=subprocess.DEVNULL, check=True) -def wait_for_container_cmd(container_name: str, cmd_to_run: [str], timeout: int): +def wait_for_container_cmd(container_name: str, cmd_to_run: List[str], timeout: int): container_exec_cmd = ["docker", "exec", container_name] cmd = container_exec_cmd + cmd_to_run @@ -129,7 +127,7 @@ def wait_for_container_cmd(container_name: str, cmd_to_run: [str], timeout: int) break time.sleep(1) - cmd_str = " ".join(cmd_to_run) + cmd_str = shlex.join(cmd_to_run) logger.error(f"Timeout while waiting for command {cmd_str} to run after {timeout} seconds") return False @@ -138,181 +136,140 @@ def start_db(clp_config: CLPConfig, conf_dir: pathlib.Path): component_name = DB_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - db_data_dir = clp_config.data_directory / component_name - db_logs_dir = clp_config.logs_directory / component_name - - validate_db_config(clp_config, db_data_dir, db_logs_dir) - - # Create directories - db_data_dir.mkdir(exist_ok=True, parents=True) - db_logs_dir.mkdir(exist_ok=True, parents=True) - - # Start container - env_dict["CLP_HOST_DB_CONF_DIR"] = str(conf_dir / "mysql" / "conf.d") - env_dict["CLP_HOST_DB_DATA_DIR"] = str(db_data_dir) - env_dict["CLP_HOST_DB_LOGS_DIR"] = str(db_logs_dir) - - # TODO: Consider a Docker Compose overwrite / extend approach for pick the right image. - if "mysql" == clp_config.database.type: - env_dict["CLP_DB_IMAGE"] = "mysql:8.0.23" - elif "mariadb" == clp_config.database.type: - env_dict["CLP_DB_IMAGE"] = "mariadb:10-jammy" + conf_file = conf_dir / "mysql" / "conf.d" / "logging.cnf" + data_dir = clp_config.data_directory / component_name + logs_dir = clp_config.logs_directory / component_name + validate_db_config(clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) + logs_dir.mkdir(exist_ok=True, parents=True) - env_dict["CLP_DB_HOST"] = get_ip_from_hostname(clp_config.database.host) - env_dict["CLP_DB_PORT"] = str(clp_config.database.port) - env_dict["CLP_DB_NAME"] = clp_config.database.name - env_dict["CLP_DB_USER"] = clp_config.database.username - env_dict["CLP_DB_PASS"] = clp_config.database.password + return { + **clp_config.database.dump_to_env_vars_dict(), + "CLP_DB_CONF_FILE_HOST": str(conf_file), + "CLP_DB_DATA_DIR_HOST": str(data_dir), + "CLP_DB_LOGS_DIR_HOST": str(logs_dir), + } def start_queue(clp_config: CLPConfig): component_name = QUEUE_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - queue_logs_dir = clp_config.logs_directory / component_name - validate_queue_config(clp_config, queue_logs_dir) - - env_dict["CLP_QUEUE_HOST"] = get_ip_from_hostname(clp_config.queue.host) - env_dict["CLP_QUEUE_PORT"] = str(clp_config.queue.port) - env_dict["CLP_QUEUE_USER"] = clp_config.queue.username - env_dict["CLP_QUEUE_PASS"] = clp_config.queue.password - - # Create directories - queue_logs_dir.mkdir(exist_ok=True, parents=True) + logs_dir = clp_config.logs_directory / component_name + validate_queue_config(clp_config, logs_dir) + logs_dir.mkdir(exist_ok=True, parents=True) - env_dict["CLP_HOST_QUEUE_LOGS_DIR"] = str(queue_logs_dir) + return { + **clp_config.queue.dump_to_env_vars_dict(), + "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), + } def start_redis(clp_config: CLPConfig, conf_dir: pathlib.Path): component_name = REDIS_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - redis_logs_dir = clp_config.logs_directory / component_name - redis_data_dir = clp_config.data_directory / component_name - - config_file_path = conf_dir / "redis" / "redis.conf" - validate_redis_config(clp_config, redis_data_dir, redis_logs_dir, config_file_path) - - env_dict["CLP_HOST_REDIS_CONF_PATH"] = str(config_file_path) - env_dict["CLP_HOST_REDIS_DATA_DIR"] = str(redis_data_dir) - env_dict["CLP_HOST_REDIS_LOGS_DIR"] = str(redis_logs_dir) - - redis_data_dir.mkdir(exist_ok=True, parents=True) - redis_logs_dir.mkdir(exist_ok=True, parents=True) + conf_file = conf_dir / "redis" / "redis.conf" + logs_dir = clp_config.logs_directory / component_name + data_dir = clp_config.data_directory / component_name + validate_redis_config(clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) + logs_dir.mkdir(exist_ok=True, parents=True) - env_dict["CLP_REDIS_HOST"] = get_ip_from_hostname(clp_config.redis.host) - env_dict["CLP_REDIS_PORT"] = str(clp_config.redis.port) - env_dict["CLP_REDIS_PASS"] = clp_config.redis.password - env_dict["CLP_REDIS_QUERY_BACKEND_DB"] = str(clp_config.redis.query_backend_database) - env_dict["CLP_REDIS_COMPRESSION_BACKEND_DB"] = str( - clp_config.redis.compression_backend_database - ) + return { + **clp_config.redis.dump_to_env_vars_dict(), + "CLP_REDIS_CONF_FILE_HOST": str(conf_file), + "CLP_REDIS_DATA_DIR_HOST": str(data_dir), + "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), + } -def start_results_cache(clp_config: CLPConfig, conf_dir: pathlib.Path): +def start_results_cache(clp_config: CLPConfig, conf_file: pathlib.Path): component_name = RESULTS_CACHE_COMPONENT_NAME logger.info(f"Initializing {component_name}...") + conf_file = conf_file / "mongo" / "mongod.conf" data_dir = clp_config.data_directory / component_name logs_dir = clp_config.logs_directory / component_name - - validate_results_cache_config(clp_config, data_dir, logs_dir) - + validate_results_cache_config(clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) - env_dict["CLP_HOST_RESULTS_CACHE_CONF_DIR"] = str(conf_dir / "mongo") - env_dict["CLP_HOST_RESULTS_CACHE_DATA_DIR"] = str(data_dir) - env_dict["CLP_HOST_RESULTS_CACHE_LOGS_DIR"] = str(logs_dir) - - env_dict["CLP_RESULTS_CACHE_HOST"] = get_ip_from_hostname(clp_config.results_cache.host) - env_dict["CLP_RESULTS_CACHE_PORT"] = str(clp_config.results_cache.port) - env_dict["CLP_RESULTS_CACHE_DB_NAME"] = clp_config.results_cache.db_name - env_dict["CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME"] = ( - clp_config.results_cache.stream_collection_name - ) + return { + **clp_config.results_cache.dump_to_env_vars_dict(), + "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), + "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), + "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), + } -def start_compression_scheduler( - clp_config: CLPConfig, -): +def start_compression_scheduler(clp_config: CLPConfig): component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - logs_dir = clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) + logs_file = clp_config.logs_directory / f"{component_name}.log" + logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) - env_dict["CLP_HOST_COMPRESSION_SCHEDULER_LOGS_DIR"] = str(logs_dir) - env_dict["CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL"] = ( - clp_config.compression_scheduler.logging_level - ) + return { + "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": clp_config.compression_scheduler.logging_level, + "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), + } -def start_query_scheduler( - clp_config: CLPConfig, -): +def start_query_scheduler(clp_config: CLPConfig): component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - logs_dir = clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) + logs_file = clp_config.logs_directory / f"{component_name}.log" + logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) - env_dict["CLP_HOST_QUERY_SCHEDULER_LOGS_DIR"] = str(logs_dir) - env_dict["CLP_QUERY_SCHEDULER_LOGGING_LEVEL"] = clp_config.query_scheduler.logging_level + return { + "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": clp_config.query_scheduler.logging_level, + "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), + } -def start_compression_worker( - clp_config: CLPConfig, - num_cpus: int, -): +def start_compression_worker(clp_config: CLPConfig, num_cpus: int): component_name = COMPRESSION_WORKER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - env_dict["CLP_COMPRESSION_WORKER_LOGS_DIR"] = str(logs_dir) - env_dict["CLP_COMPRESSION_WORKER_LOGGING_LEVEL"] = clp_config.compression_worker.logging_level - env_dict["CLP_COMPRESSION_WORKER_CONCURRENCY"] = str(num_cpus) - - # Create necessary directories - clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) - clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) + return { + "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_cpus), + "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": clp_config.compression_worker.logging_level, + "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), + } -def start_query_worker( - clp_config: CLPConfig, - num_cpus: int, -): +def start_query_worker(clp_config: CLPConfig, num_cpus: int): component_name = QUERY_WORKER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - env_dict["CLP_QUERY_WORKER_LOGS_DIR"] = str(logs_dir) - env_dict["CLP_QUERY_WORKER_LOGGING_LEVEL"] = clp_config.query_worker.logging_level - env_dict["CLP_QUERY_WORKER_CONCURRENCY"] = str(num_cpus) - - # Create necessary directories - clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) - clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) + return { + "CLP_QUERY_WORKER_LOGGING_LEVEL": clp_config.query_worker.logging_level, + "CLP_QUERY_WORKER_LOGS_DIR": str(logs_dir), + "CLP_QUERY_WORKER_CONCURRENCY": str(num_cpus), + } -def start_reducer( - clp_config: CLPConfig, - num_workers: int, -): +def start_reducer(clp_config: CLPConfig, num_workers: int): component_name = REDUCER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - env_dict["CLP_REDUCER_LOGS_DIR"] = str(logs_dir) - env_dict["CLP_REDUCER_LOGGING_LEVEL"] = clp_config.reducer.logging_level - env_dict["CLP_REDUCER_CONCURRENCY"] = str(num_workers) - env_dict["CLP_REDUCER_UPSERT_INTERVAL"] = str(clp_config.reducer.upsert_interval) + return { + "CLP_REDUCER_LOGGING_LEVEL": clp_config.reducer.logging_level, + "CLP_REDUCER_LOGS_DIR": str(logs_dir), + "CLP_REDUCER_CONCURRENCY": str(num_workers), + "CLP_REDUCER_UPSERT_INTERVAL": str(clp_config.reducer.upsert_interval), + } def update_settings_object( @@ -352,10 +309,7 @@ def read_and_update_settings_json(settings_file_path: pathlib.Path, updates: Dic return settings_object -def start_webui( - clp_config: CLPConfig, - container_clp_config: CLPConfig, -): +def start_webui(clp_config: CLPConfig, container_clp_config: CLPConfig): component_name = WEBUI_COMPONENT_NAME logger.info(f"Initializing {component_name}...") @@ -437,21 +391,17 @@ def start_webui( with open(server_settings_json_path, "w") as settings_json_file: settings_json_file.write(json.dumps(server_settings_json)) - env_dict["CLP_WEBUI_HOST"] = get_ip_from_hostname(clp_config.webui.host) - env_dict["CLP_WEBUI_PORT"] = clp_config.webui.port - env_dict["CLP_WEBUI_RATE_LIMIT"] = clp_config.webui.rate_limit + return clp_config.webui.dump_to_env_vars_dict() -def start_garbage_collector( - clp_config: CLPConfig, -): +def start_garbage_collector(clp_config: CLPConfig): component_name = GARBAGE_COLLECTOR_COMPONENT_NAME logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - env_dict["CLP_GC_LOGGING_LEVEL"] = clp_config.garbage_collector.logging_level + return clp_config.garbage_collector.dump_to_env_vars_dict() def add_num_workers_argument(parser): @@ -510,53 +460,31 @@ def main(argv): # Create necessary directories clp_config.data_directory.mkdir(parents=True, exist_ok=True) clp_config.logs_directory.mkdir(parents=True, exist_ok=True) + clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) + clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) dump_shared_container_config(container_clp_config, clp_config) - env_dict["CLP_PACKAGE_CONTAINER"] = "clp-package:dev" - - env_dict["CLP_USER_ID"] = os.getuid() - env_dict["CLP_GROUP_ID"] = os.getgid() - env_dict["CLP_STORAGE_ENGINE"] = clp_config.package.storage_engine - - env_dict["CLP_HOST_DATA_DIR"] = str(clp_config.data_directory) - env_dict["CLP_HOST_LOGS_DIR"] = str(clp_config.logs_directory) - env_dict["CLP_HOST_ARCHIVE_OUTPUT_DIR"] = str(clp_config.archive_output.get_directory()) - env_dict["CLP_HOST_STREAM_OUTPUT_DIR"] = str(clp_config.stream_output.get_directory()) - - # TODO: validate if those are required - env_dict["CLP_AWS_ACCESS_KEY_ID"] = os.getenv("AWS_ACCESS_KEY_ID", "") - env_dict["CLP_AWS_SECRET_ACCESS_KEY"] = os.getenv("AWS_SECRET_ACCESS_KEY", "") - - if clp_config.aws_config_directory is not None: - env_dict["CLP_HOST_AWS_CONFIG_DIR"] = str(clp_config.aws_config_directory) - if StorageType.S3 == clp_config.archive_output.storage.type: - clp_config.archive_output.storage.staging_directory.mkdir(parents=True, exist_ok=True) - env_dict["CLP_HOST_ARCHIVE_STAGING_DIR"] = str( - clp_config.archive_output.storage.staging_directory - ) - if StorageType.S3 == clp_config.stream_output.storage.type: - clp_config.stream_output.storage.staging_directory.mkdir(parents=True, exist_ok=True) - env_dict["CLP_HOST_STREAM_STAGING_DIR"] = str( - clp_config.stream_output.storage.staging_directory - ) - try: conf_dir = clp_home / "etc" - # Start components - start_db(clp_config, conf_dir) - start_queue(clp_config) - start_redis(clp_config, conf_dir) - start_results_cache(clp_config, conf_dir) - start_compression_scheduler(clp_config) - start_query_scheduler(clp_config) - start_compression_worker(clp_config, num_workers) - start_query_worker(clp_config, num_workers) - start_reducer(clp_config, num_workers) - start_webui(clp_config, container_clp_config) - start_garbage_collector(clp_config) - + env_dict = { + **clp_config.dump_to_env_vars_dict(), + **start_db(clp_config, conf_dir), + **start_queue(clp_config), + **start_redis(clp_config, conf_dir), + **start_results_cache(clp_config, conf_dir), + **start_compression_scheduler(clp_config), + **start_query_scheduler(clp_config), + **start_compression_worker(clp_config, num_workers), + **start_query_worker(clp_config, num_workers), + **start_reducer(clp_config, num_workers), + **start_webui(clp_config, container_clp_config), + **start_garbage_collector(clp_config), + **start_reducer(clp_config, num_workers), + **start_webui(clp_config, container_clp_config), + **start_garbage_collector(clp_config), + } with open(f"{clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): env_file.write(f"{key}={value}\n") diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index fc51ae3149..7c53b8f940 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,7 +1,8 @@ import os import pathlib +import socket from enum import auto -from typing import Literal, Optional, Union +from typing import Dict, Literal, Optional, Union from dotenv import dotenv_values from pydantic import BaseModel, PrivateAttr, root_validator, validator @@ -129,6 +130,11 @@ def validate_query_engine_package_compatibility(cls, values): return values + def dump_to_env_vars(self): + return { + "CLP_PACKAGE_STORAGE_ENGINE": self.storage_engine, + } + class Database(BaseModel): type: str = "mariadb" @@ -214,6 +220,19 @@ def get_clp_connection_params_and_type(self, disable_localhost_socket_connection def dump_to_primitive_dict(self): return self.dict(exclude={"username", "password"}) + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + return { + "CLP_DB_HOST": get_ip_from_hostname(self.host), + "CLP_DB_PORT": str(self.port), + "CLP_DB_NAME": self.name, + "CLP_DB_USER": self.username, + "CLP_DB_PASS": self.password, + "CLP_DB_IMAGE": "mysql:8.0.23" if "mysql" == self.type else "mariadb:10-jammy", + } + def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: @@ -267,8 +286,8 @@ def validate_logging_level(cls, field): class QueryScheduler(BaseModel): - host = "localhost" - port = 7000 + host: str = "localhost" + port: int = 7000 jobs_poll_delay: float = 0.1 # seconds num_archives_to_search_per_sub_job: int = 16 logging_level: str = "INFO" @@ -326,6 +345,18 @@ def validate_host(cls, field): def dump_to_primitive_dict(self): return self.dict(exclude={"password"}) + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + return { + "CLP_REDIS_HOST": get_ip_from_hostname(self.host), + "CLP_REDIS_PORT": str(self.port), + "CLP_REDIS_PASS": self.password, + "CLP_REDIS_QUERY_BACKEND_DB": str(self.query_backend_database), + "CLP_REDIS_COMPRESSION_BACKEND_DB": str(self.compression_backend_database), + } + def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: @@ -407,6 +438,17 @@ def validate_retention_period(cls, field): raise ValueError("retention_period must be greater than 0") return field + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + return { + "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(self.host), + "CLP_RESULTS_CACHE_PORT": str(self.port), + "CLP_RESULTS_CACHE_DB_NAME": self.db_name, + "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.stream_collection_name, + } + def get_uri(self): return f"mongodb://{self.host}:{self.port}/{self.db_name}" @@ -421,6 +463,17 @@ class Queue(BaseModel): def dump_to_primitive_dict(self): return self.dict(exclude={"username", "password"}) + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + return { + "CLP_QUEUE_HOST": get_ip_from_hostname(self.host), + "CLP_QUEUE_PORT": str(self.port), + "CLP_QUEUE_USER": self.username, + "CLP_QUEUE_PASS": self.password, + } + def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: @@ -722,6 +775,16 @@ def validate_rate_limit(cls, field): raise ValueError(f"rate_limit must be greater than 0") return field + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + return { + "CLP_WEBUI_HOST": get_ip_from_hostname(self.host), + "CLP_WEBUI_PORT": str(self.port), + "CLP_WEBUI_RATE_LIMIT": str(self.rate_limit), + } + class SweepInterval(BaseModel): archive: int = 60 @@ -748,6 +811,12 @@ def validate_logging_level(cls, field): _validate_logging_level(cls, field) return field + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + return {"CLP_GC_LOGGING_LEVEL": self.logging_level} + def _get_env_var(name: str) -> str: value = os.getenv(name) @@ -756,6 +825,16 @@ def _get_env_var(name: str) -> str: return value +def get_ip_from_hostname(hostname: str) -> str: + """ + Resolves a hostname to an IP address. + + :param hostname: The hostname to resolve. + :return: The resolved IP address. + """ + return socket.gethostbyname(hostname) + + class CLPConfig(BaseModel): execution_container: Optional[str] = None @@ -921,6 +1000,43 @@ def dump_to_primitive_dict(self): return d + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + env_vars = { + **self.package.dump_to_env_vars(), + # User and group IDs + "CLP_USER_ID": str(os.getuid()), + "CLP_GROUP_ID": str(os.getgid()), + # Package container + "CLP_PACKAGE_CONTAINER": "clp-package:dev", + # Global paths + "CLP_DATA_DIR_HOST": str(self.data_directory), + "CLP_LOGS_DIR_HOST": str(self.logs_directory), + "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self.archive_output.get_directory()), + "CLP_STREAM_OUTPUT_DIR_HOST": str(self.stream_output.get_directory()), + # AWS credentials + "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), + "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), + } + + # AWS config directory + if self.aws_config_directory is not None: + env_vars["CLP_AWS_CONFIG_DIR_HOST"] = str(self.aws_config_directory) + + # Staging directories for S3 storage + if StorageType.S3 == self.archive_output.storage.type: + env_vars["CLP_ARCHIVE_STAGING_DIR_HOST"] = str( + self.archive_output.storage.staging_directory + ) + if StorageType.S3 == self.stream_output.storage.type: + env_vars["CLP_STREAM_STAGING_DIR_HOST"] = str( + self.stream_output.storage.staging_directory + ) + + return env_vars + class WorkerConfig(BaseModel): package: Package = Package() diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index d3ea05493f..b76224a969 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -33,9 +33,9 @@ services: ports: - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" volumes: - - "${CLP_HOST_DATA_DIR:-./var/data}/database:/var/lib/mysql" - - "${CLP_HOST_LOGS_DIR:-./var/log}/database:/var/log/mysql" - - "./etc/mysql/conf.d/logging.cnf:/etc/mysql/conf.d/logging.cnf:ro" + - "${CLP_DB_CONF_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" + - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" + - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" healthcheck: <<: *healthcheck_defaults test: [ @@ -56,7 +56,7 @@ services: CLP_DB_USER: "${CLP_DB_USER}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" depends_on: db: condition: "service_healthy" @@ -65,7 +65,7 @@ services: "-u", "-m", "clp_py_utils.create-db-tables", "--config", "/etc/clp-config.yml", - "--storage-engine", "${CLP_STORAGE_ENGINE}" + "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE}" ] queue: @@ -79,7 +79,7 @@ services: ports: - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/queue:/var/log/rabbitmq" + - "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}:/var/log/rabbitmq" healthcheck: <<: *healthcheck_defaults test: [ @@ -94,9 +94,9 @@ services: ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" volumes: - - "${CLP_HOST_DATA_DIR:-./var/data}/redis:/data" - - "${CLP_HOST_LOGS_DIR:-./var/log}/redis:/var/log/redis" - - "./etc/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro" + - "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}:/usr/local/etc/redis/redis.conf:ro" + - "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}:/data" + - "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}:/var/log/redis" healthcheck: <<: *healthcheck_defaults test: [ @@ -120,9 +120,9 @@ services: ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" volumes: - - "${CLP_HOST_DATA_DIR:-./var/data}/results_cache:/data/db" - - "${CLP_HOST_LOGS_DIR:-./var/log}/results_cache:/var/log/mongodb" - - "./etc/mongo:/etc/mongo:ro" + - "${CLP_RESULTS_CACHE_CONF_DIR_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" + - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" + - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" healthcheck: <<: *healthcheck_defaults test: >- @@ -164,9 +164,9 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}:/var/log/compression_scheduler.log" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" depends_on: db-table-creator: @@ -196,8 +196,8 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_QUERY_SCHEDULER_LOGS_FILE_HOST:-./var/log/query_scheduler.log}:/var/log/query_scheduler.log" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -238,12 +238,12 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_HOST_ARCHIVE_STAGING_DIR:-./var/data/staged-archives}:/var/data/staged-archives" - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - - "${CLP_HOST_DATA_DIR:-./var/data}:/var/data" - - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/compression_worker:/var/log/compression_worker" + - "${CLP_ARCHIVE_STAGING_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:/var/log/compression_worker" + - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" + - "${CLP_DATA_DIR_HOST:-./var/data}/archives:/var/data/archives" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" command: [ "python3", @@ -275,12 +275,12 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" - - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/query_worker:/var/log/query_worker" - - "${CLP_HOST_STREAM_STAGING_DIR:-./var/data/staged-streams}:/var/data/staged-streams" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_DATA_DIR_HOST:-./var/data}/archives:/var/data/archives" + - "${CLP_DATA_DIR_HOST:-./var/data}/streams:/var/data/streams" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_QUERY_WORKER_LOGS_DIR:-./var/log/query_worker}:/var/log/query_worker" + - "${CLP_STREAM_STAGING_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" command: [ "python3", "-u", @@ -304,8 +304,8 @@ services: CLP_LOGS_DIR: "/var/log/reducer" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/reducer:/var/log/reducer" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_REDUCER_LOGS_DIR:-./var/log/reducer}:/var/log/reducer" depends_on: query-scheduler: condition: "service_healthy" @@ -334,8 +334,8 @@ services: ports: - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" volumes: - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_DATA_DIR_HOST:-./var/data}/streams:/var/data/streams" - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - "./var/www/webui/server/dist/settings.json\ :/opt/clp/var/www/webui/server/dist/settings.json:ro" @@ -369,8 +369,8 @@ services: CLP_LOGS_DIR: "/var/log/garbage_collector" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/garbage_collector:/var/log/garbage_collector" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" depends_on: db-table-creator: condition: "service_completed_successfully" From 60994ee02d23600c17733bc0776203017017b2c1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 00:23:42 -0400 Subject: [PATCH 031/238] fix: use List[str] type hint for command parameter in start_clp.py --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 8cf3b63bc7..53e367dbe6 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -89,7 +89,7 @@ def append_docker_options( def append_docker_port_settings_for_host_ips( - hostname: str, host_port: int, container_port: int, cmd: [str] + hostname: str, host_port: int, container_port: int, cmd: List[str] ): # Note: We use a set because gethostbyname_ex can return the same IP twice for one hostname for ip in set(socket.gethostbyname_ex(hostname)[2]): From 7e25d7556226cb5dba9a466908445d591077c1ec Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 00:48:08 -0400 Subject: [PATCH 032/238] refactor: remove `dump_to_env_vars_dict` methods and centralize environment variable handling --- .../clp_package_utils/scripts/start_clp.py | 81 ++++++++++++-- .../clp-py-utils/clp_py_utils/clp_config.py | 104 +----------------- 2 files changed, 73 insertions(+), 112 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 53e367dbe6..0f8171a559 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -2,6 +2,7 @@ import json import logging import multiprocessing +import os import pathlib import shlex import socket @@ -63,6 +64,16 @@ logger = logging.getLogger(__file__) +def get_ip_from_hostname(hostname: str) -> str: + """ + Resolves a hostname to an IP address. + + :param hostname: The hostname to resolve. + :return: The resolved IP address. + """ + return socket.gethostbyname(hostname) + + def append_docker_options( cmd: List[str], mounts: Optional[List[Optional[DockerMount]]] = None, @@ -144,10 +155,16 @@ def start_db(clp_config: CLPConfig, conf_dir: pathlib.Path): logs_dir.mkdir(exist_ok=True, parents=True) return { - **clp_config.database.dump_to_env_vars_dict(), "CLP_DB_CONF_FILE_HOST": str(conf_file), "CLP_DB_DATA_DIR_HOST": str(data_dir), "CLP_DB_LOGS_DIR_HOST": str(logs_dir), + + "CLP_DB_HOST": get_ip_from_hostname(clp_config.database.host), + "CLP_DB_PORT": str(clp_config.database.port), + "CLP_DB_NAME": clp_config.database.name, + "CLP_DB_USER": clp_config.database.username, + "CLP_DB_PASS": clp_config.database.password, + "CLP_DB_IMAGE": "mysql:8.0.23" if "mysql" == clp_config.database.type else "mariadb:10-jammy", } @@ -160,8 +177,12 @@ def start_queue(clp_config: CLPConfig): logs_dir.mkdir(exist_ok=True, parents=True) return { - **clp_config.queue.dump_to_env_vars_dict(), "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), + + "CLP_QUEUE_HOST": get_ip_from_hostname(clp_config.queue.host), + "CLP_QUEUE_PORT": str(clp_config.queue.port), + "CLP_QUEUE_USER": clp_config.queue.username, + "CLP_QUEUE_PASS": clp_config.queue.password, } @@ -177,10 +198,15 @@ def start_redis(clp_config: CLPConfig, conf_dir: pathlib.Path): logs_dir.mkdir(exist_ok=True, parents=True) return { - **clp_config.redis.dump_to_env_vars_dict(), "CLP_REDIS_CONF_FILE_HOST": str(conf_file), "CLP_REDIS_DATA_DIR_HOST": str(data_dir), "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), + + "CLP_REDIS_HOST": get_ip_from_hostname(clp_config.redis.host), + "CLP_REDIS_PORT": str(clp_config.redis.port), + "CLP_REDIS_PASS": clp_config.redis.password, + "CLP_REDIS_QUERY_BACKEND_DB": str(clp_config.redis.query_backend_database), + "CLP_REDIS_COMPRESSION_BACKEND_DB": str(clp_config.redis.compression_backend_database), } @@ -196,10 +222,14 @@ def start_results_cache(clp_config: CLPConfig, conf_file: pathlib.Path): logs_dir.mkdir(exist_ok=True, parents=True) return { - **clp_config.results_cache.dump_to_env_vars_dict(), "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), + + "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(clp_config.results_cache.host), + "CLP_RESULTS_CACHE_PORT": str(clp_config.results_cache.port), + "CLP_RESULTS_CACHE_DB_NAME": clp_config.results_cache.db_name, + "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": clp_config.results_cache.stream_collection_name, } @@ -391,7 +421,11 @@ def start_webui(clp_config: CLPConfig, container_clp_config: CLPConfig): with open(server_settings_json_path, "w") as settings_json_file: settings_json_file.write(json.dumps(server_settings_json)) - return clp_config.webui.dump_to_env_vars_dict() + return { + "CLP_WEBUI_HOST": get_ip_from_hostname(clp_config.webui.host), + "CLP_WEBUI_PORT": str(clp_config.webui.port), + "CLP_WEBUI_RATE_LIMIT": str(clp_config.webui.rate_limit), + } def start_garbage_collector(clp_config: CLPConfig): @@ -401,8 +435,9 @@ def start_garbage_collector(clp_config: CLPConfig): logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return clp_config.garbage_collector.dump_to_env_vars_dict() - + return { + "CLP_GC_LOGGING_LEVEL": clp_config.garbage_collector.logging_level + } def add_num_workers_argument(parser): parser.add_argument( @@ -469,7 +504,21 @@ def main(argv): conf_dir = clp_home / "etc" env_dict = { - **clp_config.dump_to_env_vars_dict(), + "CLP_PACKAGE_STORAGE_ENGINE": clp_config.package.storage_engine, + # User and group IDs + "CLP_USER_ID": str(os.getuid()), + "CLP_GROUP_ID": str(os.getgid()), + # Package container + "CLP_PACKAGE_CONTAINER": "clp-package:dev", + # Global paths + "CLP_DATA_DIR_HOST": str(clp_config.data_directory), + "CLP_LOGS_DIR_HOST": str(clp_config.logs_directory), + "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(clp_config.archive_output.get_directory()), + "CLP_STREAM_OUTPUT_DIR_HOST": str(clp_config.stream_output.get_directory()), + # AWS credentials + "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), + "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), + **start_db(clp_config, conf_dir), **start_queue(clp_config), **start_redis(clp_config, conf_dir), @@ -481,10 +530,20 @@ def main(argv): **start_reducer(clp_config, num_workers), **start_webui(clp_config, container_clp_config), **start_garbage_collector(clp_config), - **start_reducer(clp_config, num_workers), - **start_webui(clp_config, container_clp_config), - **start_garbage_collector(clp_config), } + # AWS config directory + if clp_config.aws_config_directory is not None: + env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(clp_config.aws_config_directory) + + # Staging directories for S3 storage + if StorageType.S3 == clp_config.archive_output.storage.type: + env_dict["CLP_ARCHIVE_STAGING_DIR_HOST"] = str( + clp_config.archive_output.storage.staging_directory + ) + if StorageType.S3 == clp_config.stream_output.storage.type: + env_dict["CLP_STREAM_STAGING_DIR_HOST"] = str( + clp_config.stream_output.storage.staging_directory + ) with open(f"{clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): env_file.write(f"{key}={value}\n") diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 7c53b8f940..7bec8926ad 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -130,11 +130,6 @@ def validate_query_engine_package_compatibility(cls, values): return values - def dump_to_env_vars(self): - return { - "CLP_PACKAGE_STORAGE_ENGINE": self.storage_engine, - } - class Database(BaseModel): type: str = "mariadb" @@ -220,18 +215,7 @@ def get_clp_connection_params_and_type(self, disable_localhost_socket_connection def dump_to_primitive_dict(self): return self.dict(exclude={"username", "password"}) - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - return { - "CLP_DB_HOST": get_ip_from_hostname(self.host), - "CLP_DB_PORT": str(self.port), - "CLP_DB_NAME": self.name, - "CLP_DB_USER": self.username, - "CLP_DB_PASS": self.password, - "CLP_DB_IMAGE": "mysql:8.0.23" if "mysql" == self.type else "mariadb:10-jammy", - } + def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) @@ -345,17 +329,7 @@ def validate_host(cls, field): def dump_to_primitive_dict(self): return self.dict(exclude={"password"}) - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - return { - "CLP_REDIS_HOST": get_ip_from_hostname(self.host), - "CLP_REDIS_PORT": str(self.port), - "CLP_REDIS_PASS": self.password, - "CLP_REDIS_QUERY_BACKEND_DB": str(self.query_backend_database), - "CLP_REDIS_COMPRESSION_BACKEND_DB": str(self.compression_backend_database), - } + def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) @@ -438,16 +412,7 @@ def validate_retention_period(cls, field): raise ValueError("retention_period must be greater than 0") return field - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - return { - "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(self.host), - "CLP_RESULTS_CACHE_PORT": str(self.port), - "CLP_RESULTS_CACHE_DB_NAME": self.db_name, - "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.stream_collection_name, - } + def get_uri(self): return f"mongodb://{self.host}:{self.port}/{self.db_name}" @@ -463,16 +428,6 @@ class Queue(BaseModel): def dump_to_primitive_dict(self): return self.dict(exclude={"username", "password"}) - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - return { - "CLP_QUEUE_HOST": get_ip_from_hostname(self.host), - "CLP_QUEUE_PORT": str(self.port), - "CLP_QUEUE_USER": self.username, - "CLP_QUEUE_PASS": self.password, - } def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) @@ -775,16 +730,6 @@ def validate_rate_limit(cls, field): raise ValueError(f"rate_limit must be greater than 0") return field - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - return { - "CLP_WEBUI_HOST": get_ip_from_hostname(self.host), - "CLP_WEBUI_PORT": str(self.port), - "CLP_WEBUI_RATE_LIMIT": str(self.rate_limit), - } - class SweepInterval(BaseModel): archive: int = 60 @@ -811,12 +756,6 @@ def validate_logging_level(cls, field): _validate_logging_level(cls, field) return field - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - return {"CLP_GC_LOGGING_LEVEL": self.logging_level} - def _get_env_var(name: str) -> str: value = os.getenv(name) @@ -1000,43 +939,6 @@ def dump_to_primitive_dict(self): return d - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - env_vars = { - **self.package.dump_to_env_vars(), - # User and group IDs - "CLP_USER_ID": str(os.getuid()), - "CLP_GROUP_ID": str(os.getgid()), - # Package container - "CLP_PACKAGE_CONTAINER": "clp-package:dev", - # Global paths - "CLP_DATA_DIR_HOST": str(self.data_directory), - "CLP_LOGS_DIR_HOST": str(self.logs_directory), - "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self.archive_output.get_directory()), - "CLP_STREAM_OUTPUT_DIR_HOST": str(self.stream_output.get_directory()), - # AWS credentials - "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), - "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - } - - # AWS config directory - if self.aws_config_directory is not None: - env_vars["CLP_AWS_CONFIG_DIR_HOST"] = str(self.aws_config_directory) - - # Staging directories for S3 storage - if StorageType.S3 == self.archive_output.storage.type: - env_vars["CLP_ARCHIVE_STAGING_DIR_HOST"] = str( - self.archive_output.storage.staging_directory - ) - if StorageType.S3 == self.stream_output.storage.type: - env_vars["CLP_STREAM_STAGING_DIR_HOST"] = str( - self.stream_output.storage.staging_directory - ) - - return env_vars - class WorkerConfig(BaseModel): package: Package = Package() From 3e24e4e2b63ed0d9b312385013bb4eeeb7103226 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 00:48:32 -0400 Subject: [PATCH 033/238] lint --- .../clp_package_utils/scripts/start_clp.py | 14 +++++--------- components/clp-py-utils/clp_py_utils/clp_config.py | 7 ------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 0f8171a559..9abfdf9a14 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -158,13 +158,14 @@ def start_db(clp_config: CLPConfig, conf_dir: pathlib.Path): "CLP_DB_CONF_FILE_HOST": str(conf_file), "CLP_DB_DATA_DIR_HOST": str(data_dir), "CLP_DB_LOGS_DIR_HOST": str(logs_dir), - "CLP_DB_HOST": get_ip_from_hostname(clp_config.database.host), "CLP_DB_PORT": str(clp_config.database.port), "CLP_DB_NAME": clp_config.database.name, "CLP_DB_USER": clp_config.database.username, "CLP_DB_PASS": clp_config.database.password, - "CLP_DB_IMAGE": "mysql:8.0.23" if "mysql" == clp_config.database.type else "mariadb:10-jammy", + "CLP_DB_IMAGE": ( + "mysql:8.0.23" if "mysql" == clp_config.database.type else "mariadb:10-jammy" + ), } @@ -178,7 +179,6 @@ def start_queue(clp_config: CLPConfig): return { "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), - "CLP_QUEUE_HOST": get_ip_from_hostname(clp_config.queue.host), "CLP_QUEUE_PORT": str(clp_config.queue.port), "CLP_QUEUE_USER": clp_config.queue.username, @@ -201,7 +201,6 @@ def start_redis(clp_config: CLPConfig, conf_dir: pathlib.Path): "CLP_REDIS_CONF_FILE_HOST": str(conf_file), "CLP_REDIS_DATA_DIR_HOST": str(data_dir), "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), - "CLP_REDIS_HOST": get_ip_from_hostname(clp_config.redis.host), "CLP_REDIS_PORT": str(clp_config.redis.port), "CLP_REDIS_PASS": clp_config.redis.password, @@ -225,7 +224,6 @@ def start_results_cache(clp_config: CLPConfig, conf_file: pathlib.Path): "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), - "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(clp_config.results_cache.host), "CLP_RESULTS_CACHE_PORT": str(clp_config.results_cache.port), "CLP_RESULTS_CACHE_DB_NAME": clp_config.results_cache.db_name, @@ -435,9 +433,8 @@ def start_garbage_collector(clp_config: CLPConfig): logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return { - "CLP_GC_LOGGING_LEVEL": clp_config.garbage_collector.logging_level - } + return {"CLP_GC_LOGGING_LEVEL": clp_config.garbage_collector.logging_level} + def add_num_workers_argument(parser): parser.add_argument( @@ -518,7 +515,6 @@ def main(argv): # AWS credentials "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - **start_db(clp_config, conf_dir), **start_queue(clp_config), **start_redis(clp_config, conf_dir), diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 7bec8926ad..abef21babc 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -215,8 +215,6 @@ def get_clp_connection_params_and_type(self, disable_localhost_socket_connection def dump_to_primitive_dict(self): return self.dict(exclude={"username", "password"}) - - def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: @@ -329,8 +327,6 @@ def validate_host(cls, field): def dump_to_primitive_dict(self): return self.dict(exclude={"password"}) - - def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: @@ -412,8 +408,6 @@ def validate_retention_period(cls, field): raise ValueError("retention_period must be greater than 0") return field - - def get_uri(self): return f"mongodb://{self.host}:{self.port}/{self.db_name}" @@ -428,7 +422,6 @@ class Queue(BaseModel): def dump_to_primitive_dict(self): return self.dict(exclude={"username", "password"}) - def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: From cbb9ce1f537be96525a47087ec228acbbb5f3b7b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 01:47:30 -0400 Subject: [PATCH 034/238] refactor: modularize and simplify start_clp.py by introducing DockerComposeController --- .../clp_package_utils/controllers.py | 455 ++++++++++++++++ .../clp_package_utils/general.py | 6 +- .../clp_package_utils/scripts/start_clp.py | 500 +----------------- .../clp_package_utils/scripts/stop_clp.py | 4 +- 4 files changed, 463 insertions(+), 502 deletions(-) create mode 100644 components/clp-package-utils/clp_package_utils/controllers.py diff --git a/components/clp-package-utils/clp_package_utils/controllers.py b/components/clp-package-utils/clp_package_utils/controllers.py new file mode 100644 index 0000000000..880f1f7266 --- /dev/null +++ b/components/clp-package-utils/clp_package_utils/controllers.py @@ -0,0 +1,455 @@ +import json +import logging +import multiprocessing +import os +import pathlib +import socket +import stat +import subprocess +from abc import ABC, abstractmethod +from typing import Any, Dict + +from clp_py_utils.clp_config import ( + AwsAuthType, + CLPConfig, + COMPRESSION_SCHEDULER_COMPONENT_NAME, + COMPRESSION_WORKER_COMPONENT_NAME, + DB_COMPONENT_NAME, + GARBAGE_COLLECTOR_COMPONENT_NAME, + QUERY_SCHEDULER_COMPONENT_NAME, + QUERY_WORKER_COMPONENT_NAME, + QUEUE_COMPONENT_NAME, + REDIS_COMPONENT_NAME, + REDUCER_COMPONENT_NAME, + RESULTS_CACHE_COMPONENT_NAME, + StorageEngine, + StorageType, + WEBUI_COMPONENT_NAME, +) +from clp_py_utils.clp_metadata_db_utils import ( + get_archives_table_name, + get_datasets_table_name, + get_files_table_name, +) + +from clp_package_utils.general import ( + check_docker_dependencies, + CONTAINER_CLP_HOME, + generate_docker_compose_container_config, + get_clp_home, + validate_db_config, + validate_queue_config, + validate_redis_config, + validate_results_cache_config, + validate_webui_config, +) + +LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + +logger = logging.getLogger(__name__) + + +def get_ip_from_hostname(hostname: str) -> str: + """ + Resolves a hostname to an IP address. + + :param hostname: The hostname to resolve. + :return: The resolved IP address. + """ + return socket.gethostbyname(hostname) + + +class BaseController(ABC): + def __init__(self, clp_config: CLPConfig): + self.clp_config = clp_config + self.clp_home = get_clp_home() + + def provision_database(self, conf_dir: pathlib.Path): + component_name = DB_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + conf_file = conf_dir / "mysql" / "conf.d" / "logging.cnf" + data_dir = self.clp_config.data_directory / component_name + logs_dir = self.clp_config.logs_directory / component_name + validate_db_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) + logs_dir.mkdir(exist_ok=True, parents=True) + + return { + "CLP_DB_CONF_FILE_HOST": str(conf_file), + "CLP_DB_DATA_DIR_HOST": str(data_dir), + "CLP_DB_LOGS_DIR_HOST": str(logs_dir), + "CLP_DB_HOST": get_ip_from_hostname(self.clp_config.database.host), + "CLP_DB_PORT": str(self.clp_config.database.port), + "CLP_DB_NAME": self.clp_config.database.name, + "CLP_DB_USER": self.clp_config.database.username, + "CLP_DB_PASS": self.clp_config.database.password, + "CLP_DB_IMAGE": ( + "mysql:8.0.23" if "mysql" == self.clp_config.database.type else "mariadb:10-jammy" + ), + } + + def provision_queue(self): + component_name = QUEUE_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_dir = self.clp_config.logs_directory / component_name + validate_queue_config(self.clp_config, logs_dir) + logs_dir.mkdir(exist_ok=True, parents=True) + + return { + "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), + "CLP_QUEUE_HOST": get_ip_from_hostname(self.clp_config.queue.host), + "CLP_QUEUE_PORT": str(self.clp_config.queue.port), + "CLP_QUEUE_USER": self.clp_config.queue.username, + "CLP_QUEUE_PASS": self.clp_config.queue.password, + } + + def provision_redis(self, conf_dir: pathlib.Path): + component_name = REDIS_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + conf_file = conf_dir / "redis" / "redis.conf" + logs_dir = self.clp_config.logs_directory / component_name + data_dir = self.clp_config.data_directory / component_name + validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) + logs_dir.mkdir(exist_ok=True, parents=True) + + return { + "CLP_REDIS_CONF_FILE_HOST": str(conf_file), + "CLP_REDIS_DATA_DIR_HOST": str(data_dir), + "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), + "CLP_REDIS_HOST": get_ip_from_hostname(self.clp_config.redis.host), + "CLP_REDIS_PORT": str(self.clp_config.redis.port), + "CLP_REDIS_PASS": self.clp_config.redis.password, + "CLP_REDIS_QUERY_BACKEND_DB": str(self.clp_config.redis.query_backend_database), + "CLP_REDIS_COMPRESSION_BACKEND_DB": str( + self.clp_config.redis.compression_backend_database + ), + } + + def provision_results_cache(self, conf_dir: pathlib.Path): + component_name = RESULTS_CACHE_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + conf_file = conf_dir / "mongo" / "mongod.conf" + data_dir = self.clp_config.data_directory / component_name + logs_dir = self.clp_config.logs_directory / component_name + validate_results_cache_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) + logs_dir.mkdir(exist_ok=True, parents=True) + + return { + "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), + "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), + "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), + "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(self.clp_config.results_cache.host), + "CLP_RESULTS_CACHE_PORT": str(self.clp_config.results_cache.port), + "CLP_RESULTS_CACHE_DB_NAME": self.clp_config.results_cache.db_name, + "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.clp_config.results_cache.stream_collection_name, + } + + def provision_compression_scheduler(self): + component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_file = self.clp_config.logs_directory / f"{component_name}.log" + logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) + + return { + "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self.clp_config.compression_scheduler.logging_level, + "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), + } + + def provision_query_scheduler(self): + component_name = QUERY_SCHEDULER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_file = self.clp_config.logs_directory / f"{component_name}.log" + logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) + + return { + "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self.clp_config.query_scheduler.logging_level, + "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), + } + + def provision_compression_worker(self, num_workers: int): + component_name = COMPRESSION_WORKER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) + + return { + "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_workers), + "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": self.clp_config.compression_worker.logging_level, + "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), + } + + def provision_query_worker(self, num_workers: int): + component_name = QUERY_WORKER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) + + return { + "CLP_QUERY_WORKER_LOGGING_LEVEL": self.clp_config.query_worker.logging_level, + "CLP_QUERY_WORKER_LOGS_DIR": str(logs_dir), + "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), + } + + def provision_reducer(self, num_workers: int): + component_name = REDUCER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) + + return { + "CLP_REDUCER_LOGGING_LEVEL": self.clp_config.reducer.logging_level, + "CLP_REDUCER_LOGS_DIR": str(logs_dir), + "CLP_REDUCER_CONCURRENCY": str(num_workers), + "CLP_REDUCER_UPSERT_INTERVAL": str(self.clp_config.reducer.upsert_interval), + } + + def _update_settings_object( + self, + parent_key_prefix: str, + settings: Dict[str, Any], + updates: Dict[str, Any], + ): + """ + Recursively updates the given settings object with the values from `updates`. + + :param parent_key_prefix: The prefix for keys at this level in the settings dictionary. + :param settings: The settings to update. + :param updates: The updates. + :raises ValueError: If a key in `updates` doesn't exist in `settings`. + """ + for key, value in updates.items(): + if key not in settings: + error_msg = ( + f"{parent_key_prefix}{key} is not a valid configuration key for the webui." + ) + raise ValueError(error_msg) + if isinstance(value, dict): + self._update_settings_object(f"{parent_key_prefix}{key}.", settings[key], value) + else: + settings[key] = updates[key] + + def _read_and_update_settings_json( + self, settings_file_path: pathlib.Path, updates: Dict[str, Any] + ): + """ + Reads and updates a settings JSON file. + + :param settings_file_path: + :param updates: + """ + with open(settings_file_path, "r") as settings_json_file: + settings_object = json.loads(settings_json_file.read()) + self._update_settings_object("", settings_object, updates) + + return settings_object + + def provision_webui(self, container_clp_config: CLPConfig): + component_name = WEBUI_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" + client_settings_json_path = ( + self.clp_home / "var" / "www" / "webui" / "client" / "settings.json" + ) + server_settings_json_path = ( + self.clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" + ) + + validate_webui_config(self.clp_config, client_settings_json_path, server_settings_json_path) + + # Read, update, and write back client's and server's settings.json + clp_db_connection_params = self.clp_config.database.get_clp_connection_params_and_type(True) + table_prefix = clp_db_connection_params["table_prefix"] + if StorageEngine.CLP_S == self.clp_config.package.storage_engine: + archives_table_name = "" + files_table_name = "" + else: + archives_table_name = get_archives_table_name(table_prefix, None) + files_table_name = get_files_table_name(table_prefix, None) + + client_settings_json_updates = { + "ClpStorageEngine": self.clp_config.package.storage_engine, + "ClpQueryEngine": self.clp_config.package.query_engine, + "MongoDbSearchResultsMetadataCollectionName": self.clp_config.webui.results_metadata_collection_name, + "SqlDbClpArchivesTableName": archives_table_name, + "SqlDbClpDatasetsTableName": get_datasets_table_name(table_prefix), + "SqlDbClpFilesTableName": files_table_name, + "SqlDbClpTablePrefix": table_prefix, + "SqlDbCompressionJobsTableName": "compression_jobs", + } + client_settings_json = self._read_and_update_settings_json( + client_settings_json_path, client_settings_json_updates + ) + with open(client_settings_json_path, "w") as client_settings_json_file: + client_settings_json_file.write(json.dumps(client_settings_json)) + + server_settings_json_updates = { + "SqlDbHost": container_clp_config.database.host, + "SqlDbPort": self.clp_config.database.port, + "SqlDbName": self.clp_config.database.name, + "SqlDbQueryJobsTableName": "query_jobs", + "MongoDbHost": container_clp_config.results_cache.host, + "MongoDbPort": self.clp_config.results_cache.port, + "MongoDbName": self.clp_config.results_cache.db_name, + "MongoDbSearchResultsMetadataCollectionName": self.clp_config.webui.results_metadata_collection_name, + "MongoDbStreamFilesCollectionName": self.clp_config.results_cache.stream_collection_name, + "ClientDir": str(container_webui_dir / "client"), + "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), + "StreamTargetUncompressedSize": self.clp_config.stream_output.target_uncompressed_size, + } + + stream_storage = self.clp_config.stream_output.storage + if StorageType.S3 == stream_storage.type: + s3_config = stream_storage.s3_config + server_settings_json_updates["StreamFilesDir"] = None + server_settings_json_updates["StreamFilesS3Region"] = s3_config.region_code + server_settings_json_updates["StreamFilesS3PathPrefix"] = ( + f"{s3_config.bucket}/{s3_config.key_prefix}" + ) + auth = s3_config.aws_authentication + if AwsAuthType.profile == auth.type: + server_settings_json_updates["StreamFilesS3Profile"] = auth.profile + else: + server_settings_json_updates["StreamFilesS3Profile"] = None + elif StorageType.FS == stream_storage.type: + server_settings_json_updates["StreamFilesDir"] = str( + container_clp_config.stream_output.get_directory() + ) + server_settings_json_updates["StreamFilesS3Region"] = None + server_settings_json_updates["StreamFilesS3PathPrefix"] = None + server_settings_json_updates["StreamFilesS3Profile"] = None + + server_settings_json = self._read_and_update_settings_json( + server_settings_json_path, server_settings_json_updates + ) + with open(server_settings_json_path, "w") as settings_json_file: + settings_json_file.write(json.dumps(server_settings_json)) + + return { + "CLP_WEBUI_HOST": get_ip_from_hostname(self.clp_config.webui.host), + "CLP_WEBUI_PORT": str(self.clp_config.webui.port), + "CLP_WEBUI_RATE_LIMIT": str(self.clp_config.webui.rate_limit), + } + + def provision_garbage_collector(self): + component_name = GARBAGE_COLLECTOR_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) + + return {"CLP_GC_LOGGING_LEVEL": self.clp_config.garbage_collector.logging_level} + + @abstractmethod + def deploy(self): + """ + Deploys the provisioned components with orchestrator-specific logic. + """ + pass + + @abstractmethod + def _provision(self) -> Dict[str, str]: + """ + Provisions all components with orchestrator-specific logic. + + :return: Dictionary of environment variables for the orchestrator + """ + pass + + +class DockerComposeController(BaseController): + @staticmethod + def _get_num_workers(): + """ + Gets the parallelism number for worker components. + TODO: Revisit after moving from single-container to multi-container workers. + """ + return multiprocessing.cpu_count() // 2 + + def __init__(self, clp_config: CLPConfig): + super().__init__(clp_config) + check_docker_dependencies(should_compose_run=False) + + def deploy(self): + self._provision() + + # Start Docker Compose + logger.info(f"Starting CLP using Docker Compose...") + try: + subprocess.run( + ["docker", "compose", "up", "-d"], + cwd=self.clp_home, + stderr=subprocess.STDOUT, + check=True, + ) + except subprocess.CalledProcessError: + logger.exception("Failed to start CLP.") + raise + + def _provision(self): + # Create necessary directories + self.clp_config.data_directory.mkdir(parents=True, exist_ok=True) + self.clp_config.logs_directory.mkdir(parents=True, exist_ok=True) + self.clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) + self.clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) + + container_clp_config = generate_docker_compose_container_config(self.clp_config) + conf_dir = self.clp_home / "etc" + num_workers = self._get_num_workers() + + env_dict = { + "CLP_PACKAGE_STORAGE_ENGINE": self.clp_config.package.storage_engine, + # User and group IDs + "CLP_USER_ID": str(os.getuid()), + "CLP_GROUP_ID": str(os.getgid()), + # Package container + "CLP_PACKAGE_CONTAINER": "clp-package:dev", + # Global paths + "CLP_DATA_DIR_HOST": str(self.clp_config.data_directory), + "CLP_LOGS_DIR_HOST": str(self.clp_config.logs_directory), + "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self.clp_config.archive_output.get_directory()), + "CLP_STREAM_OUTPUT_DIR_HOST": str(self.clp_config.stream_output.get_directory()), + # AWS credentials + "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), + "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), + **self.provision_database(conf_dir), + **self.provision_queue(), + **self.provision_redis(conf_dir), + **self.provision_results_cache(conf_dir), + **self.provision_compression_scheduler(), + **self.provision_query_scheduler(), + **self.provision_compression_worker(num_workers), + **self.provision_query_worker(num_workers), + **self.provision_reducer(num_workers), + **self.provision_webui(container_clp_config), + **self.provision_garbage_collector(), + } + + # AWS config directory + if self.clp_config.aws_config_directory is not None: + env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self.clp_config.aws_config_directory) + + # Staging directories for S3 storage + if StorageType.S3 == self.clp_config.archive_output.storage.type: + env_dict["CLP_ARCHIVE_STAGING_DIR_HOST"] = str( + self.clp_config.archive_output.storage.staging_directory + ) + if StorageType.S3 == self.clp_config.stream_output.storage.type: + env_dict["CLP_STREAM_STAGING_DIR_HOST"] = str( + self.clp_config.stream_output.storage.staging_directory + ) + + with open(f"{self.clp_home}/.env", "w") as env_file: + for key, value in env_dict.items(): + env_file.write(f"{key}={value}\n") diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 302f145e92..20c31b265f 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -140,7 +140,7 @@ def generate_container_name(job_type: str) -> str: return f"clp-{job_type}-{str(uuid.uuid4())[-4:]}" -def is_compose_running(): +def is_docker_compose_running(): cmd = ["docker", "compose", "ls", "--quiet"] try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) @@ -149,7 +149,7 @@ def is_compose_running(): raise EnvironmentError("docker-compose is not installed or not functioning properly.") -def check_dependencies(should_compose_run: bool = False): +def check_docker_dependencies(should_compose_run: bool = False): try: subprocess.run( "command -v docker", @@ -161,7 +161,7 @@ def check_dependencies(should_compose_run: bool = False): except subprocess.CalledProcessError: raise EnvironmentError("docker is not installed or available on the path") - is_running = is_compose_running() + is_running = is_docker_compose_running() if should_compose_run and not is_running: raise EnvironmentError("docker-compose is not running.") if not should_compose_run and is_running: diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 9abfdf9a14..0b7021768a 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -1,47 +1,11 @@ import argparse -import json import logging -import multiprocessing -import os import pathlib -import shlex -import socket -import stat -import subprocess import sys -import time -from typing import Any, Dict, List, Optional - -from clp_py_utils.clp_config import ( - AwsAuthType, - CLPConfig, - COMPRESSION_JOBS_TABLE_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - DB_COMPONENT_NAME, - GARBAGE_COLLECTOR_COMPONENT_NAME, - QUERY_JOBS_TABLE_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - QUEUE_COMPONENT_NAME, - REDIS_COMPONENT_NAME, - REDUCER_COMPONENT_NAME, - RESULTS_CACHE_COMPONENT_NAME, - StorageEngine, - StorageType, - WEBUI_COMPONENT_NAME, -) -from clp_py_utils.clp_metadata_db_utils import ( - get_archives_table_name, - get_datasets_table_name, - get_files_table_name, -) +from clp_package_utils.controllers import DockerComposeController from clp_package_utils.general import ( - check_dependencies, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, - CONTAINER_CLP_HOME, - DockerMount, dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, @@ -49,402 +13,14 @@ validate_and_load_db_credentials_file, validate_and_load_queue_credentials_file, validate_and_load_redis_credentials_file, - validate_db_config, validate_logs_input_config, validate_output_storage_config, - validate_queue_config, - validate_redis_config, - validate_results_cache_config, validate_retention_config, - validate_webui_config, ) -LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH - logger = logging.getLogger(__file__) -def get_ip_from_hostname(hostname: str) -> str: - """ - Resolves a hostname to an IP address. - - :param hostname: The hostname to resolve. - :return: The resolved IP address. - """ - return socket.gethostbyname(hostname) - - -def append_docker_options( - cmd: List[str], - mounts: Optional[List[Optional[DockerMount]]] = None, - env_vars: Optional[List[str]] = None, -): - """ - Appends Docker mount and environment variable options to a command list. - - :param cmd: The command list to append options to. - :param mounts: Optional list of DockerMount objects to add as --mount options. - :param env_vars: Optional list of environment variables to add as -e options. - """ - if mounts: - for mount in mounts: - if mount: - cmd.append("--mount") - cmd.append(str(mount)) - - if env_vars: - for env_var in env_vars: - if "" != env_var: - cmd.append("-e") - cmd.append(env_var) - - -def append_docker_port_settings_for_host_ips( - hostname: str, host_port: int, container_port: int, cmd: List[str] -): - # Note: We use a set because gethostbyname_ex can return the same IP twice for one hostname - for ip in set(socket.gethostbyname_ex(hostname)[2]): - cmd.append("-p") - cmd.append(f"{ip}:{host_port}:{container_port}") - - -def chown_recursively( - path: pathlib.Path, - user_id: int, - group_id: int, -): - """ - Recursively changes the owner of the given path to the given user ID and group ID. - :param path: - :param user_id: - :param group_id: - """ - chown_cmd = ["chown", "--recursive", f"{user_id}:{group_id}", str(path)] - subprocess.run(chown_cmd, stdout=subprocess.DEVNULL, check=True) - - -def wait_for_container_cmd(container_name: str, cmd_to_run: List[str], timeout: int): - container_exec_cmd = ["docker", "exec", container_name] - cmd = container_exec_cmd + cmd_to_run - - begin_time = time.time() - - while True: - try: - subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) - return True - except subprocess.CalledProcessError: - if time.time() - begin_time > timeout: - break - time.sleep(1) - - cmd_str = shlex.join(cmd_to_run) - logger.error(f"Timeout while waiting for command {cmd_str} to run after {timeout} seconds") - return False - - -def start_db(clp_config: CLPConfig, conf_dir: pathlib.Path): - component_name = DB_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - conf_file = conf_dir / "mysql" / "conf.d" / "logging.cnf" - data_dir = clp_config.data_directory / component_name - logs_dir = clp_config.logs_directory / component_name - validate_db_config(clp_config, conf_file, data_dir, logs_dir) - data_dir.mkdir(exist_ok=True, parents=True) - logs_dir.mkdir(exist_ok=True, parents=True) - - return { - "CLP_DB_CONF_FILE_HOST": str(conf_file), - "CLP_DB_DATA_DIR_HOST": str(data_dir), - "CLP_DB_LOGS_DIR_HOST": str(logs_dir), - "CLP_DB_HOST": get_ip_from_hostname(clp_config.database.host), - "CLP_DB_PORT": str(clp_config.database.port), - "CLP_DB_NAME": clp_config.database.name, - "CLP_DB_USER": clp_config.database.username, - "CLP_DB_PASS": clp_config.database.password, - "CLP_DB_IMAGE": ( - "mysql:8.0.23" if "mysql" == clp_config.database.type else "mariadb:10-jammy" - ), - } - - -def start_queue(clp_config: CLPConfig): - component_name = QUEUE_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_dir = clp_config.logs_directory / component_name - validate_queue_config(clp_config, logs_dir) - logs_dir.mkdir(exist_ok=True, parents=True) - - return { - "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), - "CLP_QUEUE_HOST": get_ip_from_hostname(clp_config.queue.host), - "CLP_QUEUE_PORT": str(clp_config.queue.port), - "CLP_QUEUE_USER": clp_config.queue.username, - "CLP_QUEUE_PASS": clp_config.queue.password, - } - - -def start_redis(clp_config: CLPConfig, conf_dir: pathlib.Path): - component_name = REDIS_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - conf_file = conf_dir / "redis" / "redis.conf" - logs_dir = clp_config.logs_directory / component_name - data_dir = clp_config.data_directory / component_name - validate_redis_config(clp_config, conf_file, data_dir, logs_dir) - data_dir.mkdir(exist_ok=True, parents=True) - logs_dir.mkdir(exist_ok=True, parents=True) - - return { - "CLP_REDIS_CONF_FILE_HOST": str(conf_file), - "CLP_REDIS_DATA_DIR_HOST": str(data_dir), - "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), - "CLP_REDIS_HOST": get_ip_from_hostname(clp_config.redis.host), - "CLP_REDIS_PORT": str(clp_config.redis.port), - "CLP_REDIS_PASS": clp_config.redis.password, - "CLP_REDIS_QUERY_BACKEND_DB": str(clp_config.redis.query_backend_database), - "CLP_REDIS_COMPRESSION_BACKEND_DB": str(clp_config.redis.compression_backend_database), - } - - -def start_results_cache(clp_config: CLPConfig, conf_file: pathlib.Path): - component_name = RESULTS_CACHE_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - conf_file = conf_file / "mongo" / "mongod.conf" - data_dir = clp_config.data_directory / component_name - logs_dir = clp_config.logs_directory / component_name - validate_results_cache_config(clp_config, conf_file, data_dir, logs_dir) - data_dir.mkdir(exist_ok=True, parents=True) - logs_dir.mkdir(exist_ok=True, parents=True) - - return { - "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), - "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), - "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), - "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(clp_config.results_cache.host), - "CLP_RESULTS_CACHE_PORT": str(clp_config.results_cache.port), - "CLP_RESULTS_CACHE_DB_NAME": clp_config.results_cache.db_name, - "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": clp_config.results_cache.stream_collection_name, - } - - -def start_compression_scheduler(clp_config: CLPConfig): - component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_file = clp_config.logs_directory / f"{component_name}.log" - logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) - - return { - "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": clp_config.compression_scheduler.logging_level, - "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), - } - - -def start_query_scheduler(clp_config: CLPConfig): - component_name = QUERY_SCHEDULER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_file = clp_config.logs_directory / f"{component_name}.log" - logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) - - return { - "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": clp_config.query_scheduler.logging_level, - "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), - } - - -def start_compression_worker(clp_config: CLPConfig, num_cpus: int): - component_name = COMPRESSION_WORKER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_dir = clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) - - return { - "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_cpus), - "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": clp_config.compression_worker.logging_level, - "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), - } - - -def start_query_worker(clp_config: CLPConfig, num_cpus: int): - component_name = QUERY_WORKER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_dir = clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) - - return { - "CLP_QUERY_WORKER_LOGGING_LEVEL": clp_config.query_worker.logging_level, - "CLP_QUERY_WORKER_LOGS_DIR": str(logs_dir), - "CLP_QUERY_WORKER_CONCURRENCY": str(num_cpus), - } - - -def start_reducer(clp_config: CLPConfig, num_workers: int): - component_name = REDUCER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_dir = clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) - - return { - "CLP_REDUCER_LOGGING_LEVEL": clp_config.reducer.logging_level, - "CLP_REDUCER_LOGS_DIR": str(logs_dir), - "CLP_REDUCER_CONCURRENCY": str(num_workers), - "CLP_REDUCER_UPSERT_INTERVAL": str(clp_config.reducer.upsert_interval), - } - - -def update_settings_object( - parent_key_prefix: str, - settings: Dict[str, Any], - updates: Dict[str, Any], -): - """ - Recursively updates the given settings object with the values from `updates`. - - :param parent_key_prefix: The prefix for keys at this level in the settings dictionary. - :param settings: The settings to update. - :param updates: The updates. - :raises ValueError: If a key in `updates` doesn't exist in `settings`. - """ - for key, value in updates.items(): - if key not in settings: - error_msg = f"{parent_key_prefix}{key} is not a valid configuration key for the webui." - raise ValueError(error_msg) - if isinstance(value, dict): - update_settings_object(f"{parent_key_prefix}{key}.", settings[key], value) - else: - settings[key] = updates[key] - - -def read_and_update_settings_json(settings_file_path: pathlib.Path, updates: Dict[str, Any]): - """ - Reads and updates a settings JSON file. - - :param settings_file_path: - :param updates: - """ - with open(settings_file_path, "r") as settings_json_file: - settings_object = json.loads(settings_json_file.read()) - update_settings_object("", settings_object, updates) - - return settings_object - - -def start_webui(clp_config: CLPConfig, container_clp_config: CLPConfig): - component_name = WEBUI_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" - client_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "client" / "settings.json" - ) - server_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "settings.json" - ) - - validate_webui_config(clp_config, client_settings_json_path, server_settings_json_path) - - # Read, update, and write back client's and server's settings.json - clp_db_connection_params = clp_config.database.get_clp_connection_params_and_type(True) - table_prefix = clp_db_connection_params["table_prefix"] - if StorageEngine.CLP_S == clp_config.package.storage_engine: - archives_table_name = "" - files_table_name = "" - else: - archives_table_name = get_archives_table_name(table_prefix, None) - files_table_name = get_files_table_name(table_prefix, None) - - client_settings_json_updates = { - "ClpStorageEngine": clp_config.package.storage_engine, - "ClpQueryEngine": clp_config.package.query_engine, - "MongoDbSearchResultsMetadataCollectionName": clp_config.webui.results_metadata_collection_name, - "SqlDbClpArchivesTableName": archives_table_name, - "SqlDbClpDatasetsTableName": get_datasets_table_name(table_prefix), - "SqlDbClpFilesTableName": files_table_name, - "SqlDbClpTablePrefix": table_prefix, - "SqlDbCompressionJobsTableName": COMPRESSION_JOBS_TABLE_NAME, - } - client_settings_json = read_and_update_settings_json( - client_settings_json_path, client_settings_json_updates - ) - with open(client_settings_json_path, "w") as client_settings_json_file: - client_settings_json_file.write(json.dumps(client_settings_json)) - - server_settings_json_updates = { - "SqlDbHost": container_clp_config.database.host, - "SqlDbPort": clp_config.database.port, - "SqlDbName": clp_config.database.name, - "SqlDbQueryJobsTableName": QUERY_JOBS_TABLE_NAME, - "MongoDbHost": container_clp_config.results_cache.host, - "MongoDbPort": clp_config.results_cache.port, - "MongoDbName": clp_config.results_cache.db_name, - "MongoDbSearchResultsMetadataCollectionName": clp_config.webui.results_metadata_collection_name, - "MongoDbStreamFilesCollectionName": clp_config.results_cache.stream_collection_name, - "ClientDir": str(container_webui_dir / "client"), - "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), - "StreamTargetUncompressedSize": clp_config.stream_output.target_uncompressed_size, - } - - stream_storage = clp_config.stream_output.storage - if StorageType.S3 == stream_storage.type: - s3_config = stream_storage.s3_config - server_settings_json_updates["StreamFilesDir"] = None - server_settings_json_updates["StreamFilesS3Region"] = s3_config.region_code - server_settings_json_updates["StreamFilesS3PathPrefix"] = ( - f"{s3_config.bucket}/{s3_config.key_prefix}" - ) - auth = s3_config.aws_authentication - if AwsAuthType.profile == auth.type: - server_settings_json_updates["StreamFilesS3Profile"] = auth.profile - else: - server_settings_json_updates["StreamFilesS3Profile"] = None - elif StorageType.FS == stream_storage.type: - server_settings_json_updates["StreamFilesDir"] = str( - container_clp_config.stream_output.get_directory() - ) - server_settings_json_updates["StreamFilesS3Region"] = None - server_settings_json_updates["StreamFilesS3PathPrefix"] = None - server_settings_json_updates["StreamFilesS3Profile"] = None - - server_settings_json = read_and_update_settings_json( - server_settings_json_path, server_settings_json_updates - ) - with open(server_settings_json_path, "w") as settings_json_file: - settings_json_file.write(json.dumps(server_settings_json)) - - return { - "CLP_WEBUI_HOST": get_ip_from_hostname(clp_config.webui.host), - "CLP_WEBUI_PORT": str(clp_config.webui.port), - "CLP_WEBUI_RATE_LIMIT": str(clp_config.webui.rate_limit), - } - - -def start_garbage_collector(clp_config: CLPConfig): - component_name = GARBAGE_COLLECTOR_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_dir = clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) - - return {"CLP_GC_LOGGING_LEVEL": clp_config.garbage_collector.logging_level} - - -def add_num_workers_argument(parser): - parser.add_argument( - "--num-workers", - type=int, - default=multiprocessing.cpu_count(), - help="Number of workers to start", - ) - - def main(argv): clp_home = get_clp_home() default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH @@ -459,12 +35,6 @@ def main(argv): parsed_args = args_parser.parse_args(argv[1:]) - try: - check_dependencies(should_compose_run=False) - except: - logger.exception("Dependency checking failed.") - return -1 - # Validate and load config file try: config_file_path = pathlib.Path(parsed_args.config) @@ -484,65 +54,12 @@ def main(argv): logger.exception("Failed to load config.") return -1 - # TODO: Rely on Docker Compose to spawn multiple workers - num_workers = multiprocessing.cpu_count() // 2 - container_clp_config = generate_docker_compose_container_config(clp_config) - - # Create necessary directories - clp_config.data_directory.mkdir(parents=True, exist_ok=True) - clp_config.logs_directory.mkdir(parents=True, exist_ok=True) - clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) - clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) - dump_shared_container_config(container_clp_config, clp_config) try: - conf_dir = clp_home / "etc" - - env_dict = { - "CLP_PACKAGE_STORAGE_ENGINE": clp_config.package.storage_engine, - # User and group IDs - "CLP_USER_ID": str(os.getuid()), - "CLP_GROUP_ID": str(os.getgid()), - # Package container - "CLP_PACKAGE_CONTAINER": "clp-package:dev", - # Global paths - "CLP_DATA_DIR_HOST": str(clp_config.data_directory), - "CLP_LOGS_DIR_HOST": str(clp_config.logs_directory), - "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(clp_config.archive_output.get_directory()), - "CLP_STREAM_OUTPUT_DIR_HOST": str(clp_config.stream_output.get_directory()), - # AWS credentials - "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), - "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - **start_db(clp_config, conf_dir), - **start_queue(clp_config), - **start_redis(clp_config, conf_dir), - **start_results_cache(clp_config, conf_dir), - **start_compression_scheduler(clp_config), - **start_query_scheduler(clp_config), - **start_compression_worker(clp_config, num_workers), - **start_query_worker(clp_config, num_workers), - **start_reducer(clp_config, num_workers), - **start_webui(clp_config, container_clp_config), - **start_garbage_collector(clp_config), - } - # AWS config directory - if clp_config.aws_config_directory is not None: - env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(clp_config.aws_config_directory) - - # Staging directories for S3 storage - if StorageType.S3 == clp_config.archive_output.storage.type: - env_dict["CLP_ARCHIVE_STAGING_DIR_HOST"] = str( - clp_config.archive_output.storage.staging_directory - ) - if StorageType.S3 == clp_config.stream_output.storage.type: - env_dict["CLP_STREAM_STAGING_DIR_HOST"] = str( - clp_config.stream_output.storage.staging_directory - ) - with open(f"{clp_home}/.env", "w") as env_file: - for key, value in env_dict.items(): - env_file.write(f"{key}={value}\n") + controller = DockerComposeController(clp_config) + controller.deploy() except Exception as ex: if type(ex) == ValueError: logger.error(f"Failed to initialize CLP: {ex}") @@ -550,17 +67,6 @@ def main(argv): logger.exception("Failed to initialize CLP.") return -1 - logger.info(f"Starting CLP using Docker Compose...") - try: - subprocess.run( - ["docker", "compose", "up", "-d"], - stderr=subprocess.STDOUT, - check=True, - ) - except subprocess.CalledProcessError: - logger.exception("Failed to start CLP.") - return -1 - return 0 diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 39912ce87c..7cabe58ecc 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -2,14 +2,14 @@ import subprocess import sys -from clp_package_utils.general import check_dependencies +from clp_package_utils.general import check_docker_dependencies logger = logging.getLogger(__file__) def main(): try: - check_dependencies(should_compose_run=True) + check_docker_dependencies(should_compose_run=True) except: logger.exception("Dependency checking failed.") return -1 From 3eb8dfe591ad0399ebe09e84db5dff6f88e6450d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 01:48:50 -0400 Subject: [PATCH 035/238] refactor: remove obsolete node-specific directory configuration comments --- components/clp-package-utils/clp_package_utils/general.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 20c31b265f..a0292fa848 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -462,11 +462,6 @@ def load_config_file( validate_path_for_container_mount(clp_config.data_directory) validate_path_for_container_mount(clp_config.logs_directory) - # # Make data and logs directories node-specific - # hostname = socket.gethostname() - # clp_config.data_directory /= hostname - # clp_config.logs_directory /= hostname - return clp_config From 669fa9c9e6a3fd006129e301cbe0d56f255b2847 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 01:58:24 -0400 Subject: [PATCH 036/238] refactor: remove redundant `conf_dir` parameter and use centralized config directory in controller methods --- .../clp_package_utils/controllers.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controllers.py b/components/clp-package-utils/clp_package_utils/controllers.py index 880f1f7266..9ea293c2d7 100644 --- a/components/clp-package-utils/clp_package_utils/controllers.py +++ b/components/clp-package-utils/clp_package_utils/controllers.py @@ -63,12 +63,13 @@ class BaseController(ABC): def __init__(self, clp_config: CLPConfig): self.clp_config = clp_config self.clp_home = get_clp_home() + self.conf_dir = self.clp_home / "etc" - def provision_database(self, conf_dir: pathlib.Path): + def provision_database(self): component_name = DB_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - conf_file = conf_dir / "mysql" / "conf.d" / "logging.cnf" + conf_file = self.conf_dir / "mysql" / "conf.d" / "logging.cnf" data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name validate_db_config(self.clp_config, conf_file, data_dir, logs_dir) @@ -105,11 +106,11 @@ def provision_queue(self): "CLP_QUEUE_PASS": self.clp_config.queue.password, } - def provision_redis(self, conf_dir: pathlib.Path): + def provision_redis(self): component_name = REDIS_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - conf_file = conf_dir / "redis" / "redis.conf" + conf_file = self.conf_dir / "redis" / "redis.conf" logs_dir = self.clp_config.logs_directory / component_name data_dir = self.clp_config.data_directory / component_name validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) @@ -129,11 +130,11 @@ def provision_redis(self, conf_dir: pathlib.Path): ), } - def provision_results_cache(self, conf_dir: pathlib.Path): + def provision_results_cache(self): component_name = RESULTS_CACHE_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - conf_file = conf_dir / "mongo" / "mongod.conf" + conf_file = self.conf_dir / "mongo" / "mongod.conf" data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name validate_results_cache_config(self.clp_config, conf_file, data_dir, logs_dir) @@ -405,7 +406,6 @@ def _provision(self): self.clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) container_clp_config = generate_docker_compose_container_config(self.clp_config) - conf_dir = self.clp_home / "etc" num_workers = self._get_num_workers() env_dict = { @@ -423,10 +423,10 @@ def _provision(self): # AWS credentials "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - **self.provision_database(conf_dir), + **self.provision_database(), **self.provision_queue(), - **self.provision_redis(conf_dir), - **self.provision_results_cache(conf_dir), + **self.provision_redis(), + **self.provision_results_cache(), **self.provision_compression_scheduler(), **self.provision_query_scheduler(), **self.provision_compression_worker(num_workers), From acee071ed12bee4cea5b10def41df3438663eb18 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 01:58:41 -0400 Subject: [PATCH 037/238] refactor: rename `controllers` module to `controller` and update import paths in start_clp.py --- .../clp_package_utils/{controllers.py => controller.py} | 0 .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename components/clp-package-utils/clp_package_utils/{controllers.py => controller.py} (100%) diff --git a/components/clp-package-utils/clp_package_utils/controllers.py b/components/clp-package-utils/clp_package_utils/controller.py similarity index 100% rename from components/clp-package-utils/clp_package_utils/controllers.py rename to components/clp-package-utils/clp_package_utils/controller.py diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 0b7021768a..3ccc91eee6 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -3,7 +3,7 @@ import pathlib import sys -from clp_package_utils.controllers import DockerComposeController +from clp_package_utils.controller import DockerComposeController from clp_package_utils.general import ( CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, dump_shared_container_config, From 3c45cfa8572b83e165a2c42f86603e7bd0454ccd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:01:28 -0400 Subject: [PATCH 038/238] refactor: make `validate_log_directory` private and update references accordingly --- .../clp-package-utils/clp_package_utils/general.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index a0292fa848..8aa271c4ad 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -168,7 +168,7 @@ def check_docker_dependencies(should_compose_run: bool = False): raise EnvironmentError("docker-compose is already running.") -def validate_log_directory(logs_dir: pathlib.Path, component_name: str) -> None: +def _validate_log_directory(logs_dir: pathlib.Path, component_name: str) -> None: try: validate_path_could_be_dir(logs_dir) except ValueError as ex: @@ -522,13 +522,13 @@ def validate_db_config( f"{DB_COMPONENT_NAME} base configuration at {str(base_config)} is missing." ) _validate_data_directory(data_dir, DB_COMPONENT_NAME) - validate_log_directory(logs_dir, DB_COMPONENT_NAME) + _validate_log_directory(logs_dir, DB_COMPONENT_NAME) validate_port(f"{DB_COMPONENT_NAME}.port", clp_config.database.host, clp_config.database.port) def validate_queue_config(clp_config: CLPConfig, logs_dir: pathlib.Path): - validate_log_directory(logs_dir, QUEUE_COMPONENT_NAME) + _validate_log_directory(logs_dir, QUEUE_COMPONENT_NAME) validate_port(f"{QUEUE_COMPONENT_NAME}.port", clp_config.queue.host, clp_config.queue.port) @@ -541,13 +541,13 @@ def validate_redis_config( f"{REDIS_COMPONENT_NAME} base configuration at {str(base_config)} is missing." ) _validate_data_directory(data_dir, REDIS_COMPONENT_NAME) - validate_log_directory(logs_dir, REDIS_COMPONENT_NAME) + _validate_log_directory(logs_dir, REDIS_COMPONENT_NAME) validate_port(f"{REDIS_COMPONENT_NAME}.port", clp_config.redis.host, clp_config.redis.port) def validate_reducer_config(clp_config: CLPConfig, logs_dir: pathlib.Path, num_workers: int): - validate_log_directory(logs_dir, REDUCER_COMPONENT_NAME) + _validate_log_directory(logs_dir, REDUCER_COMPONENT_NAME) for i in range(0, num_workers): validate_port( @@ -565,7 +565,7 @@ def validate_results_cache_config( f"{RESULTS_CACHE_COMPONENT_NAME} base configuration at {str(base_config)} is missing." ) _validate_data_directory(data_dir, RESULTS_CACHE_COMPONENT_NAME) - validate_log_directory(logs_dir, RESULTS_CACHE_COMPONENT_NAME) + _validate_log_directory(logs_dir, RESULTS_CACHE_COMPONENT_NAME) validate_port( f"{RESULTS_CACHE_COMPONENT_NAME}.port", From ed20110f6bf2effc769d03546d7489a1592377ee Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:07:19 -0400 Subject: [PATCH 039/238] refactor: rename `conf_dir` to `_conf_dir` and update references to reflect private usage --- .../clp-package-utils/clp_package_utils/controller.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 9ea293c2d7..ae63a15a34 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -63,13 +63,13 @@ class BaseController(ABC): def __init__(self, clp_config: CLPConfig): self.clp_config = clp_config self.clp_home = get_clp_home() - self.conf_dir = self.clp_home / "etc" + self._conf_dir = self.clp_home / "etc" def provision_database(self): component_name = DB_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - conf_file = self.conf_dir / "mysql" / "conf.d" / "logging.cnf" + conf_file = self._conf_dir / "mysql" / "conf.d" / "logging.cnf" data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name validate_db_config(self.clp_config, conf_file, data_dir, logs_dir) @@ -110,7 +110,7 @@ def provision_redis(self): component_name = REDIS_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - conf_file = self.conf_dir / "redis" / "redis.conf" + conf_file = self._conf_dir / "redis" / "redis.conf" logs_dir = self.clp_config.logs_directory / component_name data_dir = self.clp_config.data_directory / component_name validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) @@ -134,7 +134,7 @@ def provision_results_cache(self): component_name = RESULTS_CACHE_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - conf_file = self.conf_dir / "mongo" / "mongod.conf" + conf_file = self._conf_dir / "mongo" / "mongod.conf" data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name validate_results_cache_config(self.clp_config, conf_file, data_dir, logs_dir) From 4d1f5aae21b40a331d037bbce7cdd6ecc4fbecea Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:21:16 -0400 Subject: [PATCH 040/238] refactor: make `get_ip_from_hostname` private and update all references --- .../clp_package_utils/controller.py | 12 ++++++------ components/clp-py-utils/clp_py_utils/clp_config.py | 10 ---------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index ae63a15a34..a7e4aa6826 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -49,7 +49,7 @@ logger = logging.getLogger(__name__) -def get_ip_from_hostname(hostname: str) -> str: +def _get_ip_from_hostname(hostname: str) -> str: """ Resolves a hostname to an IP address. @@ -80,7 +80,7 @@ def provision_database(self): "CLP_DB_CONF_FILE_HOST": str(conf_file), "CLP_DB_DATA_DIR_HOST": str(data_dir), "CLP_DB_LOGS_DIR_HOST": str(logs_dir), - "CLP_DB_HOST": get_ip_from_hostname(self.clp_config.database.host), + "CLP_DB_HOST": _get_ip_from_hostname(self.clp_config.database.host), "CLP_DB_PORT": str(self.clp_config.database.port), "CLP_DB_NAME": self.clp_config.database.name, "CLP_DB_USER": self.clp_config.database.username, @@ -100,7 +100,7 @@ def provision_queue(self): return { "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), - "CLP_QUEUE_HOST": get_ip_from_hostname(self.clp_config.queue.host), + "CLP_QUEUE_HOST": _get_ip_from_hostname(self.clp_config.queue.host), "CLP_QUEUE_PORT": str(self.clp_config.queue.port), "CLP_QUEUE_USER": self.clp_config.queue.username, "CLP_QUEUE_PASS": self.clp_config.queue.password, @@ -121,7 +121,7 @@ def provision_redis(self): "CLP_REDIS_CONF_FILE_HOST": str(conf_file), "CLP_REDIS_DATA_DIR_HOST": str(data_dir), "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), - "CLP_REDIS_HOST": get_ip_from_hostname(self.clp_config.redis.host), + "CLP_REDIS_HOST": _get_ip_from_hostname(self.clp_config.redis.host), "CLP_REDIS_PORT": str(self.clp_config.redis.port), "CLP_REDIS_PASS": self.clp_config.redis.password, "CLP_REDIS_QUERY_BACKEND_DB": str(self.clp_config.redis.query_backend_database), @@ -145,7 +145,7 @@ def provision_results_cache(self): "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), - "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(self.clp_config.results_cache.host), + "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self.clp_config.results_cache.host), "CLP_RESULTS_CACHE_PORT": str(self.clp_config.results_cache.port), "CLP_RESULTS_CACHE_DB_NAME": self.clp_config.results_cache.db_name, "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.clp_config.results_cache.stream_collection_name, @@ -338,7 +338,7 @@ def provision_webui(self, container_clp_config: CLPConfig): settings_json_file.write(json.dumps(server_settings_json)) return { - "CLP_WEBUI_HOST": get_ip_from_hostname(self.clp_config.webui.host), + "CLP_WEBUI_HOST": _get_ip_from_hostname(self.clp_config.webui.host), "CLP_WEBUI_PORT": str(self.clp_config.webui.port), "CLP_WEBUI_RATE_LIMIT": str(self.clp_config.webui.rate_limit), } diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index abef21babc..2135bab404 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -757,16 +757,6 @@ def _get_env_var(name: str) -> str: return value -def get_ip_from_hostname(hostname: str) -> str: - """ - Resolves a hostname to an IP address. - - :param hostname: The hostname to resolve. - :return: The resolved IP address. - """ - return socket.gethostbyname(hostname) - - class CLPConfig(BaseModel): execution_container: Optional[str] = None From 3a698bb91d25b481efeff97951cbebeb67555760 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:24:15 -0400 Subject: [PATCH 041/238] refactor: extract `transform_for_container_config` method to simplify container config handling --- .../clp_package_utils/general.py | 38 +------------------ .../clp-py-utils/clp_py_utils/clp_config.py | 36 ++++++++++++++++++ 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 8aa271c4ad..95266da90b 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -295,47 +295,13 @@ def generate_container_config( def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig: """ - Copies the given config and corrects mount paths and hosts for Docker Compose. + Copies the given config and transforms mount paths and hosts for Docker Compose. :param clp_config: :return: The container config and the mounts. """ container_clp_config = clp_config.copy(deep=True) - - # Set container paths - container_clp_config.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH - container_clp_config.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH - if StorageType.FS == clp_config.logs_input.type: - container_clp_config.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR - - if StorageType.FS == clp_config.archive_output.storage.type: - container_clp_config.archive_output.storage.directory = ( - pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH - ) - elif StorageType.S3 == clp_config.archive_output.storage.type: - container_clp_config.archive_output.storage.staging_directory = ( - pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH - ) - - if StorageType.FS == clp_config.stream_output.storage.type: - container_clp_config.stream_output.storage.directory = ( - pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH - ) - elif StorageType.S3 == clp_config.stream_output.storage.type: - container_clp_config.stream_output.storage.staging_directory = ( - pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH - ) - - if clp_config.aws_config_directory is not None: - container_clp_config.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY - - # Set container services' hosts - container_clp_config.database.host = DB_COMPONENT_NAME - container_clp_config.queue.host = QUEUE_COMPONENT_NAME - container_clp_config.redis.host = REDIS_COMPONENT_NAME - container_clp_config.results_cache.host = RESULTS_CACHE_COMPONENT_NAME - container_clp_config.query_scheduler.host = QUERY_SCHEDULER_COMPONENT_NAME - container_clp_config.reducer.host = REDUCER_COMPONENT_NAME + container_clp_config.transform_for_container_config() return container_clp_config diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 2135bab404..4bd13aae5f 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, PrivateAttr, root_validator, validator from strenum import KebabCaseStrEnum, LowercaseStrEnum +from clp_package_utils.general import CONTAINER_AWS_CONFIG_DIRECTORY, CONTAINER_INPUT_LOGS_ROOT_DIR from .clp_logging import get_valid_logging_level, is_valid_logging_level from .core import ( get_config_value, @@ -922,6 +923,41 @@ def dump_to_primitive_dict(self): return d + def transform_for_container_config(self): + self.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH + self.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH + if StorageType.FS == self.logs_input.type: + self.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR + + if StorageType.FS == self.archive_output.storage.type: + self.archive_output.storage.directory = ( + pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + ) + elif StorageType.S3 == self.archive_output.storage.type: + self.archive_output.storage.staging_directory = ( + pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + ) + + if StorageType.FS == self.stream_output.storage.type: + self.stream_output.storage.directory = ( + pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH + ) + elif StorageType.S3 == self.stream_output.storage.type: + self.stream_output.storage.staging_directory = ( + pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + ) + + if self.aws_config_directory is not None: + self.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY + + # Set container services' hosts + self.database.host = DB_COMPONENT_NAME + self.queue.host = QUEUE_COMPONENT_NAME + self.redis.host = REDIS_COMPONENT_NAME + self.results_cache.host = RESULTS_CACHE_COMPONENT_NAME + self.query_scheduler.host = QUERY_SCHEDULER_COMPONENT_NAME + self.reducer.host = REDUCER_COMPONENT_NAME + class WorkerConfig(BaseModel): package: Package = Package() From cca84f491529bac205d99b3e0c16fcd8d732fe8a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:39:54 -0400 Subject: [PATCH 042/238] refactor: centralize and simplify path definitions and container config handling --- .../clp_package_utils/general.py | 15 +------ .../clp_package_utils/scripts/start_clp.py | 2 +- .../clp-py-utils/clp_py_utils/clp_config.py | 45 +++++++++---------- 3 files changed, 23 insertions(+), 39 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 95266da90b..575c321f14 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -13,17 +13,10 @@ import yaml from clp_py_utils.clp_config import ( - CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH, - CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH, CLP_DEFAULT_CREDENTIALS_FILE_PATH, - CLP_DEFAULT_DATA_DIRECTORY_PATH, - CLP_DEFAULT_LOG_DIRECTORY_PATH, - CLP_DEFAULT_STREAM_DIRECTORY_PATH, - CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH, CLP_SHARED_CONFIG_FILENAME, CLPConfig, DB_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, QueryEngine, QUEUE_COMPONENT_NAME, REDIS_COMPONENT_NAME, @@ -31,7 +24,7 @@ RESULTS_CACHE_COMPONENT_NAME, StorageType, WEBUI_COMPONENT_NAME, - WorkerConfig, + WorkerConfig, CONTAINER_CLP_HOME, CONTAINER_INPUT_LOGS_ROOT_DIR, CONTAINER_AWS_CONFIG_DIRECTORY, ) from clp_py_utils.clp_metadata_db_utils import ( MYSQL_TABLE_NAME_MAX_LEN, @@ -50,12 +43,6 @@ EXTRACT_IR_CMD = "i" EXTRACT_JSON_CMD = "j" -# Paths -CONTAINER_AWS_CONFIG_DIRECTORY = pathlib.Path("/") / ".aws" -CONTAINER_CLP_HOME = pathlib.Path("/") / "opt" / "clp" -CONTAINER_INPUT_LOGS_ROOT_DIR = pathlib.Path("/") / "mnt" / "logs" -CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH = pathlib.Path("etc") / "clp-config.yml" - DOCKER_MOUNT_TYPE_STRINGS = ["bind"] diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 3ccc91eee6..501d593dbb 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -5,7 +5,6 @@ from clp_package_utils.controller import DockerComposeController from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, @@ -17,6 +16,7 @@ validate_output_storage_config, validate_retention_config, ) +from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH logger = logging.getLogger(__file__) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 4bd13aae5f..db932a8c67 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -8,7 +8,6 @@ from pydantic import BaseModel, PrivateAttr, root_validator, validator from strenum import KebabCaseStrEnum, LowercaseStrEnum -from clp_package_utils.general import CONTAINER_AWS_CONFIG_DIRECTORY, CONTAINER_INPUT_LOGS_ROOT_DIR from .clp_logging import get_valid_logging_level, is_valid_logging_level from .core import ( get_config_value, @@ -41,6 +40,11 @@ OS_RELEASE_FILE_PATH = pathlib.Path("etc") / "os-release" +# Paths +CONTAINER_AWS_CONFIG_DIRECTORY = pathlib.Path("/") / ".aws" +CONTAINER_CLP_HOME = pathlib.Path("/") / "opt" / "clp" +CONTAINER_INPUT_LOGS_ROOT_DIR = pathlib.Path("/") / "mnt" / "logs" +CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH = pathlib.Path("etc") / "clp-config.yml" CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path("etc") / "credentials.yml" CLP_DEFAULT_DATA_DIRECTORY_PATH = pathlib.Path("var") / "data" CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" @@ -519,6 +523,9 @@ class S3IngestionConfig(BaseModel): def dump_to_primitive_dict(self): return self.dict() + def transform_for_container_config(self): + pass + class FsStorage(BaseModel): type: Literal[StorageType.FS.value] = StorageType.FS.value @@ -574,21 +581,29 @@ def dump_to_primitive_dict(self): class FsIngestionConfig(FsStorage): directory: pathlib.Path = pathlib.Path("/") + def transform_for_container_config(self): + self.directory = CONTAINER_INPUT_LOGS_ROOT_DIR class ArchiveFsStorage(FsStorage): directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + def transform_for_container_config(self): + self.directory = pathlib.Path("/") /CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH class StreamFsStorage(FsStorage): directory: pathlib.Path = CLP_DEFAULT_STREAM_DIRECTORY_PATH + def transform_for_container_config(self): + self.directory = pathlib.Path("/") /CLP_DEFAULT_STREAM_DIRECTORY_PATH class ArchiveS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + def transform_for_container_config(self): + self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH class StreamS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH def _get_directory_from_storage_config( @@ -926,27 +941,9 @@ def dump_to_primitive_dict(self): def transform_for_container_config(self): self.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH self.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH - if StorageType.FS == self.logs_input.type: - self.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR - - if StorageType.FS == self.archive_output.storage.type: - self.archive_output.storage.directory = ( - pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH - ) - elif StorageType.S3 == self.archive_output.storage.type: - self.archive_output.storage.staging_directory = ( - pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH - ) - - if StorageType.FS == self.stream_output.storage.type: - self.stream_output.storage.directory = ( - pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH - ) - elif StorageType.S3 == self.stream_output.storage.type: - self.stream_output.storage.staging_directory = ( - pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH - ) - + self.logs_input.transform_for_container_config() + self.archive_output.storage.transform_for_container_config() + self.stream_output.storage.transform_for_container_config() if self.aws_config_directory is not None: self.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY From 537398a36b4847b105c1bf6e01435b7ca69afbf9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:45:19 -0400 Subject: [PATCH 043/238] refactor: streamline container service configuration with `transform_for_container_config` method --- .../clp_package_utils/general.py | 5 +- .../clp_package_utils/scripts/start_clp.py | 3 +- .../clp-py-utils/clp_py_utils/clp_config.py | 49 +++++++++++++------ 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 575c321f14..47d0472531 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -16,6 +16,9 @@ CLP_DEFAULT_CREDENTIALS_FILE_PATH, CLP_SHARED_CONFIG_FILENAME, CLPConfig, + CONTAINER_AWS_CONFIG_DIRECTORY, + CONTAINER_CLP_HOME, + CONTAINER_INPUT_LOGS_ROOT_DIR, DB_COMPONENT_NAME, QueryEngine, QUEUE_COMPONENT_NAME, @@ -24,7 +27,7 @@ RESULTS_CACHE_COMPONENT_NAME, StorageType, WEBUI_COMPONENT_NAME, - WorkerConfig, CONTAINER_CLP_HOME, CONTAINER_INPUT_LOGS_ROOT_DIR, CONTAINER_AWS_CONFIG_DIRECTORY, + WorkerConfig, ) from clp_py_utils.clp_metadata_db_utils import ( MYSQL_TABLE_NAME_MAX_LEN, diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 501d593dbb..62a8aef324 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -3,6 +3,8 @@ import pathlib import sys +from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH + from clp_package_utils.controller import DockerComposeController from clp_package_utils.general import ( dump_shared_container_config, @@ -16,7 +18,6 @@ validate_output_storage_config, validate_retention_config, ) -from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH logger = logging.getLogger(__file__) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index db932a8c67..21706cd5f8 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -239,6 +239,9 @@ def load_credentials_from_env(self): self.username = _get_env_var(CLP_DB_USER_ENV_VAR_NAME) self.password = _get_env_var(CLP_DB_PASS_ENV_VAR_NAME) + def transform_for_container_config(self): + self.host = DB_COMPONENT_NAME + def _validate_logging_level(cls, field): if not is_valid_logging_level(field): @@ -296,6 +299,9 @@ def validate_port(cls, field): raise ValueError(f"{field} is not greater than zero") return field + def transform_for_container_config(self): + self.host = QUERY_SCHEDULER_COMPONENT_NAME + class CompressionWorker(BaseModel): logging_level: str = "INFO" @@ -349,6 +355,9 @@ def load_credentials_from_env(self): """ self.password = _get_env_var(CLP_REDIS_PASS_ENV_VAR_NAME) + def transform_for_container_config(self): + self.host = REDIS_COMPONENT_NAME + class Reducer(BaseModel): host: str = "localhost" @@ -379,6 +388,9 @@ def validate_upsert_interval(cls, field): raise ValueError(f"{field} is not greater than zero") return field + def transform_for_container_config(self): + self.host = REDUCER_COMPONENT_NAME + class ResultsCache(BaseModel): host: str = "localhost" @@ -416,6 +428,9 @@ def validate_retention_period(cls, field): def get_uri(self): return f"mongodb://{self.host}:{self.port}/{self.db_name}" + def transform_for_container_config(self): + self.host = RESULTS_CACHE_COMPONENT_NAME + class Queue(BaseModel): host: str = "localhost" @@ -446,6 +461,9 @@ def load_credentials_from_env(self): self.username = _get_env_var(CLP_QUEUE_USER_ENV_VAR_NAME) self.password = _get_env_var(CLP_QUEUE_PASS_ENV_VAR_NAME) + def transform_for_container_config(self): + self.host = QUEUE_COMPONENT_NAME + class S3Credentials(BaseModel): access_key_id: str @@ -584,26 +602,30 @@ class FsIngestionConfig(FsStorage): def transform_for_container_config(self): self.directory = CONTAINER_INPUT_LOGS_ROOT_DIR + class ArchiveFsStorage(FsStorage): directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH def transform_for_container_config(self): - self.directory = pathlib.Path("/") /CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + self.directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + class StreamFsStorage(FsStorage): directory: pathlib.Path = CLP_DEFAULT_STREAM_DIRECTORY_PATH def transform_for_container_config(self): - self.directory = pathlib.Path("/") /CLP_DEFAULT_STREAM_DIRECTORY_PATH + self.directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH + class ArchiveS3Storage(S3Storage): - staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH def transform_for_container_config(self): - self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + class StreamS3Storage(S3Storage): - staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH def _get_directory_from_storage_config( @@ -941,19 +963,18 @@ def dump_to_primitive_dict(self): def transform_for_container_config(self): self.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH self.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH + if self.aws_config_directory is not None: + self.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY self.logs_input.transform_for_container_config() self.archive_output.storage.transform_for_container_config() self.stream_output.storage.transform_for_container_config() - if self.aws_config_directory is not None: - self.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY - # Set container services' hosts - self.database.host = DB_COMPONENT_NAME - self.queue.host = QUEUE_COMPONENT_NAME - self.redis.host = REDIS_COMPONENT_NAME - self.results_cache.host = RESULTS_CACHE_COMPONENT_NAME - self.query_scheduler.host = QUERY_SCHEDULER_COMPONENT_NAME - self.reducer.host = REDUCER_COMPONENT_NAME + self.database.transform_for_container_config() + self.queue.transform_for_container_config() + self.redis.transform_for_container_config() + self.results_cache.transform_for_container_config() + self.query_scheduler.transform_for_container_config() + self.reducer.transform_for_container_config() class WorkerConfig(BaseModel): From 42442b838a5ef1ce9a9d46c00b53bb2d82c52691 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:49:41 -0400 Subject: [PATCH 044/238] fix imports --- .../clp_package_utils/scripts/archive_manager.py | 3 +-- .../clp-package-utils/clp_package_utils/scripts/compress.py | 3 +-- .../clp_package_utils/scripts/dataset_manager.py | 2 +- .../clp-package-utils/clp_package_utils/scripts/decompress.py | 3 +-- .../clp_package_utils/scripts/native/archive_manager.py | 3 +-- .../clp_package_utils/scripts/native/compress.py | 3 +-- .../clp_package_utils/scripts/native/dataset_manager.py | 4 ++-- .../clp_package_utils/scripts/native/decompress.py | 3 +-- .../clp_package_utils/scripts/native/search.py | 3 +-- .../clp-package-utils/clp_package_utils/scripts/search.py | 3 +-- 10 files changed, 11 insertions(+), 19 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index a86543c157..8706fe86c4 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -11,11 +11,10 @@ CLP_DB_USER_ENV_VAR_NAME, CLP_DEFAULT_DATASET_NAME, StorageEngine, - StorageType, + StorageType, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLPConfig, DockerMount, dump_container_config, diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py index 8cd2e6637c..9b33bb1995 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py @@ -11,12 +11,11 @@ CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, CLP_DEFAULT_DATASET_NAME, - StorageEngine, + StorageEngine, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from job_orchestration.scheduler.job_config import InputType from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CONTAINER_INPUT_LOGS_ROOT_DIR, dump_container_config, generate_container_config, diff --git a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py index 55c280e489..60fc994878 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py @@ -12,11 +12,11 @@ CLP_DB_USER_ENV_VAR_NAME, StorageEngine, StorageType, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_py_utils.s3_utils import generate_container_auth_options from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, dump_container_config, generate_container_config, generate_container_name, diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index b7e9893487..e7ece78b31 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -12,11 +12,10 @@ CLP_DEFAULT_DATASET_NAME, CLPConfig, StorageEngine, - StorageType, + StorageType, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, DockerMount, DockerMountType, dump_container_config, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py index 79825dcac7..968660aecf 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, List, Optional -from clp_py_utils.clp_config import Database +from clp_py_utils.clp_config import Database, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH from clp_py_utils.clp_metadata_db_utils import ( delete_archives_from_metadata_db, get_archives_table_name, @@ -15,7 +15,6 @@ from clp_py_utils.sql_adapter import SQL_Adapter from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLPConfig, get_clp_home, load_config_file, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py index 6f8411dfaa..0078467b88 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py @@ -11,7 +11,7 @@ import msgpack from clp_py_utils.clp_config import ( CLPConfig, - COMPRESSION_JOBS_TABLE_NAME, + COMPRESSION_JOBS_TABLE_NAME, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_py_utils.pretty_size import pretty_size from clp_py_utils.s3_utils import parse_s3_url @@ -29,7 +29,6 @@ ) from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CONTAINER_INPUT_LOGS_ROOT_DIR, get_clp_home, load_config_file, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py index 44ab9d2a10..c6896e00e8 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py @@ -6,7 +6,8 @@ from pathlib import Path from typing import Dict, List -from clp_py_utils.clp_config import ArchiveOutput, Database, S3Config, StorageType +from clp_py_utils.clp_config import ArchiveOutput, Database, S3Config, StorageType, \ + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH from clp_py_utils.clp_metadata_db_utils import ( delete_dataset_from_metadata_db, get_datasets_table_name, @@ -15,7 +16,6 @@ from clp_py_utils.sql_adapter import SQL_Adapter from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLPConfig, get_clp_home, load_config_file, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py index 1418271c37..9c1212ff58 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py @@ -13,7 +13,7 @@ CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, CLPConfig, - Database, + Database, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_py_utils.clp_metadata_db_utils import get_files_table_name from clp_py_utils.sql_adapter import SQL_Adapter @@ -25,7 +25,6 @@ ) from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, EXTRACT_FILE_CMD, EXTRACT_IR_CMD, EXTRACT_JSON_CMD, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/search.py b/components/clp-package-utils/clp_package_utils/scripts/native/search.py index 7c3a02e7d5..326a067d88 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/search.py @@ -12,14 +12,13 @@ import pymongo from clp_py_utils.clp_config import ( Database, - ResultsCache, + ResultsCache, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_py_utils.sql_adapter import SQL_Adapter from job_orchestration.scheduler.constants import QueryJobStatus, QueryJobType from job_orchestration.scheduler.job_config import AggregationConfig, SearchJobConfig from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, get_clp_home, load_config_file, ) diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py index eb5a2118b0..fccbdae114 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/search.py @@ -10,11 +10,10 @@ CLP_DB_USER_ENV_VAR_NAME, CLP_DEFAULT_DATASET_NAME, StorageEngine, - StorageType, + StorageType, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, dump_container_config, generate_container_config, generate_container_name, From a3288ae22ffba0d24614a8337d230fc0da7ff7fa Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:51:37 -0400 Subject: [PATCH 045/238] reorganize imports --- .../clp_package_utils/scripts/archive_manager.py | 3 ++- .../clp_package_utils/scripts/compress.py | 3 ++- .../clp_package_utils/scripts/dataset_manager.py | 2 +- .../clp_package_utils/scripts/decompress.py | 3 ++- .../clp_package_utils/scripts/native/archive_manager.py | 2 +- .../clp_package_utils/scripts/native/compress.py | 5 +++-- .../clp_package_utils/scripts/native/dataset_manager.py | 9 +++++++-- .../clp_package_utils/scripts/native/decompress.py | 3 ++- .../clp_package_utils/scripts/native/search.py | 3 ++- .../clp_package_utils/scripts/search.py | 3 ++- components/clp-py-utils/clp_py_utils/clp_config.py | 3 +-- 11 files changed, 25 insertions(+), 14 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index 8706fe86c4..d137d90883 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -9,9 +9,10 @@ from clp_py_utils.clp_config import ( CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_DEFAULT_DATASET_NAME, StorageEngine, - StorageType, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + StorageType, ) from clp_package_utils.general import ( diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py index 9b33bb1995..010d4b7612 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py @@ -10,8 +10,9 @@ from clp_py_utils.clp_config import ( CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_DEFAULT_DATASET_NAME, - StorageEngine, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + StorageEngine, ) from job_orchestration.scheduler.job_config import InputType diff --git a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py index 60fc994878..7cd81766f1 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py @@ -10,9 +10,9 @@ ARCHIVE_MANAGER_ACTION_NAME, CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, StorageEngine, StorageType, - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_py_utils.s3_utils import generate_container_auth_options diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index e7ece78b31..379316007b 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -9,10 +9,11 @@ from clp_py_utils.clp_config import ( CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_DEFAULT_DATASET_NAME, CLPConfig, StorageEngine, - StorageType, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + StorageType, ) from clp_package_utils.general import ( diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py index 968660aecf..e0349e5f2c 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, List, Optional -from clp_py_utils.clp_config import Database, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH +from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, Database from clp_py_utils.clp_metadata_db_utils import ( delete_archives_from_metadata_db, get_archives_table_name, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py index 0078467b88..de4f8330a3 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py @@ -5,13 +5,14 @@ import sys import time from contextlib import closing -from typing import List, Optional, Union +from typing import List, Union import brotli import msgpack from clp_py_utils.clp_config import ( + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLPConfig, - COMPRESSION_JOBS_TABLE_NAME, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + COMPRESSION_JOBS_TABLE_NAME, ) from clp_py_utils.pretty_size import pretty_size from clp_py_utils.s3_utils import parse_s3_url diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py index c6896e00e8..4ff30cd9fd 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py @@ -6,8 +6,13 @@ from pathlib import Path from typing import Dict, List -from clp_py_utils.clp_config import ArchiveOutput, Database, S3Config, StorageType, \ - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH +from clp_py_utils.clp_config import ( + ArchiveOutput, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + Database, + S3Config, + StorageType, +) from clp_py_utils.clp_metadata_db_utils import ( delete_dataset_from_metadata_db, get_datasets_table_name, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py index 9c1212ff58..300609439e 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py @@ -12,8 +12,9 @@ from clp_py_utils.clp_config import ( CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLPConfig, - Database, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + Database, ) from clp_py_utils.clp_metadata_db_utils import get_files_table_name from clp_py_utils.sql_adapter import SQL_Adapter diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/search.py b/components/clp-package-utils/clp_package_utils/scripts/native/search.py index 326a067d88..d08d484463 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/search.py @@ -11,8 +11,9 @@ import msgpack import pymongo from clp_py_utils.clp_config import ( + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, Database, - ResultsCache, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + ResultsCache, ) from clp_py_utils.sql_adapter import SQL_Adapter from job_orchestration.scheduler.constants import QueryJobStatus, QueryJobType diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py index fccbdae114..66d63ab2d2 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/search.py @@ -8,9 +8,10 @@ from clp_py_utils.clp_config import ( CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_DEFAULT_DATASET_NAME, StorageEngine, - StorageType, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + StorageType, ) from clp_package_utils.general import ( diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 21706cd5f8..b16ffe5051 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,8 +1,7 @@ import os import pathlib -import socket from enum import auto -from typing import Dict, Literal, Optional, Union +from typing import Literal, Optional, Union from dotenv import dotenv_values from pydantic import BaseModel, PrivateAttr, root_validator, validator From 5b3677947ce97c891fb249b9b2565a133ce927c0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:53:52 -0400 Subject: [PATCH 046/238] fix: adjust volume path formatting in docker-compose --- tools/deployment/package/docker-compose.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index b76224a969..1bf99117f2 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -165,7 +165,8 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}:/var/log/compression_scheduler.log" + - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}\ +:/var/log/compression_scheduler.log" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" depends_on: @@ -197,7 +198,8 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_QUERY_SCHEDULER_LOGS_FILE_HOST:-./var/log/query_scheduler.log}:/var/log/query_scheduler.log" + - "${CLP_QUERY_SCHEDULER_LOGS_FILE_HOST:-./var/log/query_scheduler.log}:\ +/var/log/query_scheduler.log" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -240,7 +242,8 @@ services: volumes: - "${CLP_ARCHIVE_STAGING_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:/var/log/compression_worker" + - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ +/var/log/compression_worker" - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - "${CLP_DATA_DIR_HOST:-./var/data}/archives:/var/data/archives" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" From 93882affdde9a19b882f6d5a500bbe13c03fd224 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:55:45 -0400 Subject: [PATCH 047/238] fix: correct typo in comment for scheduler healthcheck in docker-compose --- tools/deployment/package/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 1bf99117f2..4364401614 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -215,7 +215,7 @@ services: ] healthcheck: <<: *healthcheck_defaults - # FIXME: need to suppressing warnings in the schduler for reading 0 out of 8 expected bytes + # FIXME: need to suppressing warnings in the scheduler for reading 0 out of 8 expected bytes test: [ "CMD", "bash", From a4d8e1f5701c45980795cea70971618594584cfd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:04:58 -0400 Subject: [PATCH 048/238] refactor: unify volume path definitions and remove redundant staging directory handling in container configuration --- .../clp_package_utils/controller.py | 10 ---------- tools/deployment/package/docker-compose.yml | 12 ++++++------ 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index a7e4aa6826..ba94d134b8 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -440,16 +440,6 @@ def _provision(self): if self.clp_config.aws_config_directory is not None: env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self.clp_config.aws_config_directory) - # Staging directories for S3 storage - if StorageType.S3 == self.clp_config.archive_output.storage.type: - env_dict["CLP_ARCHIVE_STAGING_DIR_HOST"] = str( - self.clp_config.archive_output.storage.staging_directory - ) - if StorageType.S3 == self.clp_config.stream_output.storage.type: - env_dict["CLP_STREAM_STAGING_DIR_HOST"] = str( - self.clp_config.stream_output.storage.staging_directory - ) - with open(f"{self.clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): env_file.write(f"{key}={value}\n") diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 4364401614..7b69a00585 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -240,12 +240,12 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_ARCHIVE_STAGING_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ /var/log/compression_worker" - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - - "${CLP_DATA_DIR_HOST:-./var/data}/archives:/var/data/archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" command: [ @@ -279,11 +279,11 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_DATA_DIR_HOST:-./var/data}/archives:/var/data/archives" - - "${CLP_DATA_DIR_HOST:-./var/data}/streams:/var/data/streams" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_QUERY_WORKER_LOGS_DIR:-./var/log/query_worker}:/var/log/query_worker" - - "${CLP_STREAM_STAGING_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" command: [ "python3", "-u", @@ -338,7 +338,7 @@ services: - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" volumes: - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_DATA_DIR_HOST:-./var/data}/streams:/var/data/streams" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - "./var/www/webui/server/dist/settings.json\ :/opt/clp/var/www/webui/server/dist/settings.json:ro" From 63e6d72da77112b23e51bcca88c276df1fb6e220 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:05:37 -0400 Subject: [PATCH 049/238] remove comment --- components/clp-package-utils/clp_package_utils/controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index ba94d134b8..12716c8afa 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -436,7 +436,6 @@ def _provision(self): **self.provision_garbage_collector(), } - # AWS config directory if self.clp_config.aws_config_directory is not None: env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self.clp_config.aws_config_directory) From 0531a7bd2c533e103680b29fad68e125ab2a0e13 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:10:50 -0400 Subject: [PATCH 050/238] feat: implement `stop` method in `DockerComposeController` and update stop script to use it --- .../clp_package_utils/controller.py | 26 ++++++++++++++-- .../clp_package_utils/scripts/stop_clp.py | 30 +++++++++++-------- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 12716c8afa..55bcb32b0a 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -359,6 +359,13 @@ def deploy(self): """ pass + @abstractmethod + def stop(self): + """ + Stops the deployed components with orchestrator-specific logic. + """ + pass + @abstractmethod def _provision(self) -> Dict[str, str]: """ @@ -380,12 +387,11 @@ def _get_num_workers(): def __init__(self, clp_config: CLPConfig): super().__init__(clp_config) - check_docker_dependencies(should_compose_run=False) def deploy(self): + check_docker_dependencies(should_compose_run=False) self._provision() - # Start Docker Compose logger.info(f"Starting CLP using Docker Compose...") try: subprocess.run( @@ -398,6 +404,22 @@ def deploy(self): logger.exception("Failed to start CLP.") raise + def stop(self): + check_docker_dependencies(should_compose_run=True) + + logger.info("Stopping all CLP containers using Docker Compose...") + try: + subprocess.run( + ["docker", "compose", "down"], + cwd=self.clp_home, + stderr=subprocess.STDOUT, + check=True, + ) + logger.info("All CLP containers stopped.") + except subprocess.CalledProcessError: + logger.exception("Failed to stop CLP containers using Docker Compose.") + raise + def _provision(self): # Create necessary directories self.clp_config.data_directory.mkdir(parents=True, exist_ok=True) diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 7cabe58ecc..80bdbeee10 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -1,29 +1,33 @@ import logging -import subprocess import sys -from clp_package_utils.general import check_docker_dependencies +from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH + +from clp_package_utils.controller import DockerComposeController +from clp_package_utils.general import ( + get_clp_home, + load_config_file, +) logger = logging.getLogger(__file__) def main(): + clp_home = get_clp_home() + default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH + + # Load config file try: - check_docker_dependencies(should_compose_run=True) + clp_config = load_config_file(default_config_file_path, default_config_file_path, clp_home) except: - logger.exception("Dependency checking failed.") + logger.exception("Failed to load config.") return -1 - logger.info("Stopping all CLP containers using Docker Compose...") try: - subprocess.run( - ["docker", "compose", "down"], - stderr=subprocess.STDOUT, - check=True, - ) - logger.info("All CLP containers stopped.") - except subprocess.CalledProcessError: - logger.exception("Failed to stop CLP containers using Docker Compose.") + controller = DockerComposeController(clp_config) + controller.stop() + except: + logger.exception("Failed to stop CLP.") return -1 return 0 From f93aec76c43636bb23537b6d3c266775d12f9052 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:13:10 -0400 Subject: [PATCH 051/238] refactor: reorder volume definitions in docker-compose for consistency and clarity --- tools/deployment/package/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 7b69a00585..fe42dd968f 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -240,12 +240,12 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ /var/log/compression_worker" - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" command: [ @@ -278,12 +278,12 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_QUERY_WORKER_LOGS_DIR:-./var/log/query_worker}:/var/log/query_worker" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" command: [ "python3", "-u", From 6a206285c1127d9b4b277ea041c225cd4e9a49d5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:14:38 -0400 Subject: [PATCH 052/238] revert error message in start_clp.py --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 62a8aef324..36ae82de93 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -63,9 +63,9 @@ def main(argv): controller.deploy() except Exception as ex: if type(ex) == ValueError: - logger.error(f"Failed to initialize CLP: {ex}") + logger.error(f"Failed to start CLP: {ex}") else: - logger.exception("Failed to initialize CLP.") + logger.exception("Failed to start CLP.") return -1 return 0 From 09356586078aae503da290f6a5f6a5dfb395c4fe Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:15:35 -0400 Subject: [PATCH 053/238] refactor: standardize logger messages from "Initializing" to "Provisioning" in `controller.py` --- .../clp_package_utils/controller.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 55bcb32b0a..c7c6a52ba6 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -67,7 +67,7 @@ def __init__(self, clp_config: CLPConfig): def provision_database(self): component_name = DB_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") conf_file = self._conf_dir / "mysql" / "conf.d" / "logging.cnf" data_dir = self.clp_config.data_directory / component_name @@ -92,7 +92,7 @@ def provision_database(self): def provision_queue(self): component_name = QUEUE_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_dir = self.clp_config.logs_directory / component_name validate_queue_config(self.clp_config, logs_dir) @@ -108,7 +108,7 @@ def provision_queue(self): def provision_redis(self): component_name = REDIS_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") conf_file = self._conf_dir / "redis" / "redis.conf" logs_dir = self.clp_config.logs_directory / component_name @@ -132,7 +132,7 @@ def provision_redis(self): def provision_results_cache(self): component_name = RESULTS_CACHE_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") conf_file = self._conf_dir / "mongo" / "mongod.conf" data_dir = self.clp_config.data_directory / component_name @@ -153,7 +153,7 @@ def provision_results_cache(self): def provision_compression_scheduler(self): component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_file = self.clp_config.logs_directory / f"{component_name}.log" logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) @@ -165,7 +165,7 @@ def provision_compression_scheduler(self): def provision_query_scheduler(self): component_name = QUERY_SCHEDULER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_file = self.clp_config.logs_directory / f"{component_name}.log" logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) @@ -177,7 +177,7 @@ def provision_query_scheduler(self): def provision_compression_worker(self, num_workers: int): component_name = COMPRESSION_WORKER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -190,7 +190,7 @@ def provision_compression_worker(self, num_workers: int): def provision_query_worker(self, num_workers: int): component_name = QUERY_WORKER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -203,7 +203,7 @@ def provision_query_worker(self, num_workers: int): def provision_reducer(self, num_workers: int): component_name = REDUCER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -257,7 +257,7 @@ def _read_and_update_settings_json( def provision_webui(self, container_clp_config: CLPConfig): component_name = WEBUI_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" client_settings_json_path = ( @@ -345,7 +345,7 @@ def provision_webui(self, container_clp_config: CLPConfig): def provision_garbage_collector(self): component_name = GARBAGE_COLLECTOR_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) From ad6192a4acab284132151a6be35729fe33fb7163 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:25:26 -0400 Subject: [PATCH 054/238] remove comment --- .../clp-package-utils/clp_package_utils/scripts/stop_clp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 80bdbeee10..f0d54e6de5 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -16,7 +16,6 @@ def main(): clp_home = get_clp_home() default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH - # Load config file try: clp_config = load_config_file(default_config_file_path, default_config_file_path, clp_home) except: From 9469db4ec4804c3f19df1f104e7d00a4f5c62f75 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 04:24:32 -0400 Subject: [PATCH 055/238] docs: update multi-node deployment guide and add Docker Compose design instructions --- docs/src/dev-docs/building-package.md | 5 + docs/src/dev-docs/design-docker-compose.md | 101 +++++++++++++++ docs/src/dev-docs/index.md | 1 + docs/src/user-docs/guides-multi-node.md | 141 +-------------------- docs/src/user-docs/guides-overview.md | 1 - 5 files changed, 111 insertions(+), 138 deletions(-) create mode 100644 docs/src/dev-docs/design-docker-compose.md diff --git a/docs/src/dev-docs/building-package.md b/docs/src/dev-docs/building-package.md index b212b5d3c1..0c84f1c758 100644 --- a/docs/src/dev-docs/building-package.md +++ b/docs/src/dev-docs/building-package.md @@ -78,5 +78,10 @@ task docker-images:package This will create a Docker image named `clp-package:dev`. +The package includes a `docker-compose.yml` file that can be used to deploy CLP using Docker Compose. +If you want to manually deploy with Docker Compose instead of using the package scripts, see the +[Docker Compose design][docker-compose-design] for more information. + [clp-issue-872]: https://github.com/y-scope/clp/issues/872 +[docker-compose-design]: ../dev-docs/design-docker-compose.md [Task]: https://taskfile.dev/ diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md new file mode 100644 index 0000000000..097559c74d --- /dev/null +++ b/docs/src/dev-docs/design-docker-compose.md @@ -0,0 +1,101 @@ +# Docker Compose design + +This document explains the technical details of CLP's Docker Compose implementation. + +## Overview + +The Docker Compose implementation depends on a new controller architecture with a `BaseController` +abstract class and a `DockerComposeController` implementation. + +## Architecture + +### Controller + +The implementation uses a controller pattern: + +* `BaseController` (abstract): Defines the interface for provisioning and managing CLP components. +* `DockerComposeController`: Implements the Docker Compose-specific logic. + +### Key Components + +1. **Provisioning Methods**: Each CLP component has a dedicated provisioning method in the controller: + * `provision_database()` + * `provision_queue()` + * `provision_redis()` + * `provision_results_cache()` + * `provision_compression_scheduler()` + * `provision_query_scheduler()` + * `provision_compression_worker()` + * `provision_query_worker()` + * `provision_reducer()` + * `provision_webui()` + * `provision_garbage_collector()` + +2. **Environment Generation**: The controller generates a `.env` file with all necessary environment + variables for Docker Compose. + +3. **Configuration Transformation**: The `transform_for_container_config()` method in `CLPConfig` + and related classes adapts the configuration for containerized environments. + +## Docker Compose File + +The `docker-compose.yml` file defines all services with: + +* Proper service dependencies using `depends_on` +* Health checks for critical services +* Volume mounts for persistent data +* Network configuration +* User permissions +* Resource limits + +## Deployment Process + +1. **Configuration Loading**: The start script loads and validates the CLP configuration. +2. **Provisioning**: The controller provisions all components and generates environment variables. +3. **Environment File Generation**: A `.env` file is created with all necessary variables. +4. **Docker Compose Execution**: `docker compose up -d` is executed to start all services. + +## Service Architecture + +The Docker Compose setup includes the following services: + +* **database**: MySQL/MariaDB for metadata storage +* **queue**: RabbitMQ for job queuing +* **redis**: Redis for task result storage +* **results-cache**: MongoDB for search results caching +* **compression-scheduler**: Schedules compression jobs +* **query-scheduler**: Schedules search jobs +* **compression-worker**: Executes compression tasks +* **query-worker**: Executes search tasks +* **reducer**: Handles aggregation operations +* **webui**: Web interface for CLP +* **garbage-collector**: Manages retention policies +* **db-table-creator**: Initializes database tables +* **results-cache-indices-creator**: Sets up MongoDB indices + +## Service Dependencies + +Docker Compose manages service startup order through: + +* `depends_on` directives. +* Health checks with `condition: service_healthy`. +* Init containers for one-time setup tasks (e.g., `db-table-creator`). + +## Troubleshooting + +If you encounter issues with the Docker Compose deployment: + +1. Check service status: + ```bash + docker compose ps + ``` + +2. View service logs: + ```bash + docker compose logs + ``` + +3. Validate configuration: + ```bash + docker compose config + ``` diff --git a/docs/src/dev-docs/index.md b/docs/src/dev-docs/index.md index 5372121755..e46c339e21 100644 --- a/docs/src/dev-docs/index.md +++ b/docs/src/dev-docs/index.md @@ -79,6 +79,7 @@ tooling-gh-workflows :hidden: design-project-structure +design-docker-compose design-kv-ir-streams/index design-metadata-db diff --git a/docs/src/user-docs/guides-multi-node.md b/docs/src/user-docs/guides-multi-node.md index 44078605ad..9a56f4c631 100644 --- a/docs/src/user-docs/guides-multi-node.md +++ b/docs/src/user-docs/guides-multi-node.md @@ -2,145 +2,12 @@ A multi-node deployment allows you to run CLP across a distributed set of hosts. -## Requirements - -* [Docker] - * If you're not running as root, ensure docker can be run - [without superuser privileges][docker-non-root]. -* Python 3.9 or higher -* One or more hosts networked together -* A distributed filesystem (e.g. [SeaweedFS]) accessible by all worker hosts through a filesystem - mount - * See [below](#setting-up-seaweedfs) for how to set up a simple SeaweedFS cluster. - -## Cluster overview - -The CLP package is composed of several components--controller components and worker components. In a -cluster, there should be a single instance of each controller component and one or more instances of -worker components. The tables below list the components and their functions. - -:::{table} Controller components -:align: left - -| Component | Description | -|-----------------------|-----------------------------------------------------------------| -| database | Database for archive metadata, compression jobs, and query jobs | -| queue | Task queue for schedulers | -| redis | Task result storage for workers | -| compression_scheduler | Scheduler for compression jobs | -| query_scheduler | Scheduler for search/aggregation jobs | -| results_cache | Storage for the workers to return search results to the UI | -| webui | Web server for the UI | -| garbage_collector | Background process for retention control | -::: - -:::{table} Worker components -:align: left - -| Component | Description | -|--------------------|--------------------------------------------------------------| -| compression_worker | Worker processes for compression jobs | -| query_worker | Worker processes for search/aggregation jobs | -| reducer | Reducers for performing the final stages of aggregation jobs | -::: - -:::{note} -Running additional workers increases the parallelism of compression and search/aggregation jobs. +:::{warning} +CLP now uses Docker Compose for orchestration and support for multi-node deployments is +temporarily removed. Please contact us if you need immediate support for multi-node deployments, or +stay tuned for future updates on Kubernetes Helm support. ::: -## Configuring CLP - -1. Copy `etc/credentials.template.yml` to `etc/credentials.yml`. -2. Edit `etc/credentials.yml`: - - {style=lower-alpha} - 1. Uncomment the file. - 2. Choose an appropriate username and password. - * Note that these are *new* credentials that will be used by the components. - -3. Choose which hosts you would like to use for the controller components. - * You can use a single host for all controller components. -4. Edit `etc/clp-config.yml`: - - {style=lower-alpha} - 1. Uncomment the file. - 2. Set the `host` config of each controller component to the host that you'd like to run them - on. - * If desired, you can run different controller components on different hosts. - 3. Change any of the controller components' ports that will conflict with services you already - have running. - 4. Set `archive_output.directory` to a directory on the distributed filesystem. - * Ideally, the directory should be empty or should not yet exist (CLP will create it) since - CLP will write several files and directories directly to the given directory. - -5. Download and extract the package on all nodes. -6. Copy the `credentials.yml` and `clp-config.yml` files that you created above and paste them - into `etc` on all the hosts where you extracted the package. - -## Starting CLP - -Before starting each CLP component, note that some components must be started before others. We -organize the components into groups below, where components in a group can be started in any order, -but all components in a group must be started before starting a component in the next group. - -**Group 1 components:** - -* `database` -* `queue` -* `redis` -* `results_cache` - -**Group 2 components:** - -* `compression_scheduler` -* `query_scheduler` -* `garbage_collector` - -**Group 3 components:** - -* `compression_worker` -* `query_worker` -* `reducer` - -For each component, on the host where you want to run the component, run: - -```bash -sbin/start-clp.sh -``` - -Where `` is the name of the component in the groups above. - -## Using CLP - -To learn how to compress and search your logs, check out the quick-start guide that corresponds to -the flavor of CLP you're running: - -::::{grid} 1 1 2 2 -:gutter: 2 - -:::{grid-item-card} -:link: quick-start/clp-json -Using clp-json -^^^ -How to compress and search JSON logs. -::: - -:::{grid-item-card} -:link: quick-start/clp-text -Using clp-text -^^^ -How to compress and search unstructured text logs. -::: -:::: - -## Stopping CLP - -If you need to stop the cluster, run: - -```bash -sbin/stop-clp.sh -``` - ## Setting up SeaweedFS The instructions below are for running a simple SeaweedFS cluster on a set of hosts. For other use diff --git a/docs/src/user-docs/guides-overview.md b/docs/src/user-docs/guides-overview.md index 60d7bbacd1..62f01a9835 100644 --- a/docs/src/user-docs/guides-overview.md +++ b/docs/src/user-docs/guides-overview.md @@ -32,4 +32,3 @@ Using Presto with CLP ^^^ How to use Presto to query compressed logs in CLP. ::: -:::: From 110e9fa2f05756de7839412e9b39f6fc986517af Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 04:27:47 -0400 Subject: [PATCH 056/238] docs: refine Docker Compose design doc for clarity and consistency --- docs/src/dev-docs/design-docker-compose.md | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index 097559c74d..1a3261c20d 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -18,22 +18,10 @@ The implementation uses a controller pattern: ### Key Components -1. **Provisioning Methods**: Each CLP component has a dedicated provisioning method in the controller: - * `provision_database()` - * `provision_queue()` - * `provision_redis()` - * `provision_results_cache()` - * `provision_compression_scheduler()` - * `provision_query_scheduler()` - * `provision_compression_worker()` - * `provision_query_worker()` - * `provision_reducer()` - * `provision_webui()` - * `provision_garbage_collector()` - +1. **Provisioning Methods**: Each CLP component has a dedicated provisioning method in the + controller: `provision_()`. 2. **Environment Generation**: The controller generates a `.env` file with all necessary environment variables for Docker Compose. - 3. **Configuration Transformation**: The `transform_for_container_config()` method in `CLPConfig` and related classes adapts the configuration for containerized environments. @@ -50,6 +38,8 @@ The `docker-compose.yml` file defines all services with: ## Deployment Process +The `start-clp.py` script performs the following steps: + 1. **Configuration Loading**: The start script loads and validates the CLP configuration. 2. **Provisioning**: The controller provisions all components and generates environment variables. 3. **Environment File Generation**: A `.env` file is created with all necessary variables. @@ -59,6 +49,8 @@ The `docker-compose.yml` file defines all services with: The Docker Compose setup includes the following services: +### Services + * **database**: MySQL/MariaDB for metadata storage * **queue**: RabbitMQ for job queuing * **redis**: Redis for task result storage @@ -70,6 +62,8 @@ The Docker Compose setup includes the following services: * **reducer**: Handles aggregation operations * **webui**: Web interface for CLP * **garbage-collector**: Manages retention policies + +### One-time initialization jobs * **db-table-creator**: Initializes database tables * **results-cache-indices-creator**: Sets up MongoDB indices From 045dde623204f3bb88646d03bfe9207431d80c40 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 04:40:55 -0400 Subject: [PATCH 057/238] refactor: update CLP container configuration to use execution container from config --- .../clp_package_utils/controller.py | 2 +- components/clp-py-utils/clp_py_utils/clp_config.py | 14 ++------------ components/package-template/src/etc/clp-config.yml | 3 +++ 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index c7c6a52ba6..9b4750c471 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -436,7 +436,7 @@ def _provision(self): "CLP_USER_ID": str(os.getuid()), "CLP_GROUP_ID": str(os.getgid()), # Package container - "CLP_PACKAGE_CONTAINER": "clp-package:dev", + "CLP_PACKAGE_CONTAINER": self.clp_config.execution_container, # Global paths "CLP_DATA_DIR_HOST": str(self.clp_config.data_directory), "CLP_LOGS_DIR_HOST": str(self.clp_config.logs_directory), diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index b16ffe5051..d6d3439fbd 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -917,20 +917,10 @@ def validate_aws_config_dir(self): def load_execution_container_name(self): if self.execution_container is not None: - # Accept configured value for debug purposes + # Accept configured value for releases return - os_release = dotenv_values(self._os_release_file_path) - if "ubuntu" == os_release["ID"]: - self.execution_container = ( - f"clp-execution-x86-{os_release['ID']}-{os_release['VERSION_CODENAME']}:main" - ) - else: - raise NotImplementedError( - f"Unsupported OS {os_release['ID']} in {OS_RELEASE_FILE_PATH}" - ) - - self.execution_container = "ghcr.io/y-scope/clp/" + self.execution_container + self.execution_container = "clp-package:dev" def get_shared_config_file_path(self) -> pathlib.Path: return self.logs_directory / CLP_SHARED_CONFIG_FILENAME diff --git a/components/package-template/src/etc/clp-config.yml b/components/package-template/src/etc/clp-config.yml index 8e96b179b8..d67d4f3a56 100644 --- a/components/package-template/src/etc/clp-config.yml +++ b/components/package-template/src/etc/clp-config.yml @@ -1,3 +1,6 @@ +## Docker image to use for CLP package execution. +#execution_container: "clp-package:dev" +# ## Location (e.g., directory) containing any logs you wish to compress. Must be reachable by all ## workers. #logs_input: From fa87d9281d9766e0782e6770b7ea0693590aac87 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 12:36:18 -0400 Subject: [PATCH 058/238] refactor(controller): move directory creation logic from `_provision` to `start_clp` script. --- .../clp-package-utils/clp_package_utils/controller.py | 6 ------ .../clp_package_utils/scripts/start_clp.py | 9 ++++++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 9b4750c471..19d4e2ad69 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -421,12 +421,6 @@ def stop(self): raise def _provision(self): - # Create necessary directories - self.clp_config.data_directory.mkdir(parents=True, exist_ok=True) - self.clp_config.logs_directory.mkdir(parents=True, exist_ok=True) - self.clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) - self.clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) - container_clp_config = generate_docker_compose_container_config(self.clp_config) num_workers = self._get_num_workers() diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 36ae82de93..b4e4a3449c 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -51,13 +51,16 @@ def main(argv): clp_config.validate_data_dir() clp_config.validate_logs_dir() clp_config.validate_aws_config_dir() + + # Create necessary directories + clp_config.data_directory.mkdir(parents=True, exist_ok=True) + clp_config.logs_directory.mkdir(parents=True, exist_ok=True) + clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) + clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) except: logger.exception("Failed to load config.") return -1 - container_clp_config = generate_docker_compose_container_config(clp_config) - dump_shared_container_config(container_clp_config, clp_config) - try: controller = DockerComposeController(clp_config) controller.deploy() From b6ac2c6561b91b1324970dd77f4bbce250223574 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 12:40:41 -0400 Subject: [PATCH 059/238] feat(controller): add function to dump shared container configuration in `_provision`. --- components/clp-package-utils/clp_package_utils/controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 19d4e2ad69..956a930928 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -41,7 +41,7 @@ validate_queue_config, validate_redis_config, validate_results_cache_config, - validate_webui_config, + validate_webui_config, dump_shared_container_config, ) LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH @@ -423,6 +423,7 @@ def stop(self): def _provision(self): container_clp_config = generate_docker_compose_container_config(self.clp_config) num_workers = self._get_num_workers() + dump_shared_container_config(container_clp_config, self.clp_config) env_dict = { "CLP_PACKAGE_STORAGE_ENGINE": self.clp_config.package.storage_engine, From bfd8c7ba6683f1d5b7c2742740159efcffd1ebed Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 13:13:14 -0400 Subject: [PATCH 060/238] lint --- components/clp-package-utils/clp_package_utils/controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 956a930928..d0f78f7bd9 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -35,13 +35,14 @@ from clp_package_utils.general import ( check_docker_dependencies, CONTAINER_CLP_HOME, + dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, validate_db_config, validate_queue_config, validate_redis_config, validate_results_cache_config, - validate_webui_config, dump_shared_container_config, + validate_webui_config, ) LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH From 2b7959bf45551d076a36d37a43516411a2795bab Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 13:13:27 -0400 Subject: [PATCH 061/238] refactor(clp-py-utils): update staging directory handling in S3 storage classes. --- components/clp-py-utils/clp_py_utils/clp_config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index d6d3439fbd..846b8a9b72 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -617,14 +617,17 @@ def transform_for_container_config(self): class ArchiveS3Storage(S3Storage): - staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH def transform_for_container_config(self): self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH class StreamS3Storage(S3Storage): - staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + + def transform_for_container_config(self): + self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH def _get_directory_from_storage_config( From f9eb88c8a94faf66cba954f2bfd9a5376c7fd566 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 16:05:11 -0400 Subject: [PATCH 062/238] docs(design): Enhance Docker Compose design doc with diagrams and detailed service tables. --- docs/src/dev-docs/design-docker-compose.md | 118 +++++++++++++++++---- 1 file changed, 96 insertions(+), 22 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index 1a3261c20d..2c9e25b516 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -11,12 +11,12 @@ abstract class and a `DockerComposeController` implementation. ### Controller -The implementation uses a controller pattern: +The orchestration implementation uses a controller pattern: * `BaseController` (abstract): Defines the interface for provisioning and managing CLP components. * `DockerComposeController`: Implements the Docker Compose-specific logic. -### Key Components +### Initialization 1. **Provisioning Methods**: Each CLP component has a dedicated provisioning method in the controller: `provision_()`. @@ -49,31 +49,105 @@ The `start-clp.py` script performs the following steps: The Docker Compose setup includes the following services: -### Services - -* **database**: MySQL/MariaDB for metadata storage -* **queue**: RabbitMQ for job queuing -* **redis**: Redis for task result storage -* **results-cache**: MongoDB for search results caching -* **compression-scheduler**: Schedules compression jobs -* **query-scheduler**: Schedules search jobs -* **compression-worker**: Executes compression tasks -* **query-worker**: Executes search tasks -* **reducer**: Handles aggregation operations -* **webui**: Web interface for CLP -* **garbage-collector**: Manages retention policies +:::{mermaid} +graph LR + %% Services + db["db (MySQL)"] + queue["queue (RabbitMQ)"] + redis["redis (Redis)"] + results_cache["results-cache (MongoDB)"] + compression_scheduler["compression-scheduler"] + query_scheduler["query-scheduler"] + compression_worker["compression-worker"] + query_worker["query-worker"] + reducer["reducer"] + webui["webui"] + garbage_collector["garbage-collector"] + + %% One-time jobs + db_table_creator["db-table-creator"] + results_cache_indices_creator["results-cache-indices-creator"] + + %% Dependencies + db -->|healthy| db_table_creator + results_cache -->|healthy| results_cache_indices_creator + db_table_creator -->|completed_successfully| compression_scheduler + queue -->|healthy| compression_scheduler + redis -->|healthy| compression_scheduler + db_table_creator -->|completed_successfully| query_scheduler + queue -->|healthy| query_scheduler + redis -->|healthy| query_scheduler + query_scheduler -->|healthy| reducer + results_cache_indices_creator -->|completed_successfully| reducer + db_table_creator -->|completed_successfully| webui + results_cache_indices_creator -->|completed_successfully| webui + db_table_creator -->|completed_successfully| garbage_collector + query_scheduler -->|healthy| garbage_collector + results_cache_indices_creator -->|completed_successfully| garbage_collector + + subgraph Databases + db + queue + redis + results_cache + end + + subgraph DB Migration Jobs + db_table_creator + results_cache_indices_creator + end + + subgraph Schedulers + compression_scheduler + query_scheduler + end + + subgraph Workers + compression_worker + query_worker + reducer + end + + subgraph UI & Management + webui + garbage_collector + end +::: + +### Services overview + +The CLP package is composed of several service components. The tables below list the services and their functions. + +:::{table} Services +:align: left + +| Service | Description | +|-----------------------|-----------------------------------------------------------------| +| database | Database for archive metadata, compression jobs, and query jobs | +| queue | Task queue for schedulers | +| redis | Task result storage for workers | +| compression_scheduler | Scheduler for compression jobs | +| query_scheduler | Scheduler for search/aggregation jobs | +| results_cache | Storage for the workers to return search results to the UI | +| compression_worker | Worker processes for compression jobs | +| query_worker | Worker processes for search/aggregation jobs | +| reducer | Reducers for performing the final stages of aggregation jobs | +| webui | Web server for the UI | +| garbage_collector | Background process for retention control | +::: ### One-time initialization jobs -* **db-table-creator**: Initializes database tables -* **results-cache-indices-creator**: Sets up MongoDB indices -## Service Dependencies +We also set up short-lived run-once "services" to initialize some services listed above. -Docker Compose manages service startup order through: +:::{table} Initialization jobs +:align: left -* `depends_on` directives. -* Health checks with `condition: service_healthy`. -* Init containers for one-time setup tasks (e.g., `db-table-creator`). +| Job | Description | +|-------------------------------|---------------------------------------------------------| +| db-table-creator | Initializes database tables | +| results-cache-indices-creator | Initializes single-node replica set and sets up indices | +::: ## Troubleshooting From 0f5bd45bc8f0c3023f74ffdba83ec85e5470cb39 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 16:56:58 -0400 Subject: [PATCH 063/238] fix title case --- docs/src/dev-docs/design-docker-compose.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index 2c9e25b516..10c17edd28 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -45,7 +45,7 @@ The `start-clp.py` script performs the following steps: 3. **Environment File Generation**: A `.env` file is created with all necessary variables. 4. **Docker Compose Execution**: `docker compose up -d` is executed to start all services. -## Service Architecture +## Service architecture The Docker Compose setup includes the following services: From 065692880f2583cae41bd28d9a85a1bfca97e57a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 17:05:40 -0400 Subject: [PATCH 064/238] Update references to docker-compose.yml to docker-compose.yaml. --- docs/src/dev-docs/building-package.md | 2 +- docs/src/dev-docs/design-docker-compose.md | 2 +- taskfile.yaml | 4 ++-- .../package/{docker-compose.yml => docker-compose.yaml} | 0 4 files changed, 4 insertions(+), 4 deletions(-) rename tools/deployment/package/{docker-compose.yml => docker-compose.yaml} (100%) diff --git a/docs/src/dev-docs/building-package.md b/docs/src/dev-docs/building-package.md index 0c84f1c758..c1422c00c9 100644 --- a/docs/src/dev-docs/building-package.md +++ b/docs/src/dev-docs/building-package.md @@ -78,7 +78,7 @@ task docker-images:package This will create a Docker image named `clp-package:dev`. -The package includes a `docker-compose.yml` file that can be used to deploy CLP using Docker Compose. +The package includes a `docker-compose.yaml` file that can be used to deploy CLP using Docker Compose. If you want to manually deploy with Docker Compose instead of using the package scripts, see the [Docker Compose design][docker-compose-design] for more information. diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index 10c17edd28..d845a242ae 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -27,7 +27,7 @@ The orchestration implementation uses a controller pattern: ## Docker Compose File -The `docker-compose.yml` file defines all services with: +The `docker-compose.yaml` file defines all services with: * Proper service dependencies using `depends_on` * Health checks for critical services diff --git a/taskfile.yaml b/taskfile.yaml index 90a0e886d8..933b8012dc 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -113,7 +113,7 @@ tasks: - "components/clp-py-utils/dist/*.whl" - "components/job-orchestration/dist/*.whl" - "components/package-template/src/**/*" - - "tools/deployment/package/docker-compose.yml" + - "tools/deployment/package/docker-compose.yaml" generates: ["{{.CHECKSUM_FILE}}"] deps: - "core" @@ -163,7 +163,7 @@ tasks: PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm ci --omit=dev - >- rsync -a - "tools/deployment/package/docker-compose.yml" + "tools/deployment/package/docker-compose.yaml" "{{.OUTPUT_DIR}}" # This command must be last - task: "utils:checksum:compute" diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yaml similarity index 100% rename from tools/deployment/package/docker-compose.yml rename to tools/deployment/package/docker-compose.yaml From 66dcbb21c4c5feff6841211611d6f96cad457c67 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 10 Sep 2025 13:28:42 -0400 Subject: [PATCH 065/238] feat(docker): add project name to docker-compose and enhance running check --- .../clp-package-utils/clp_package_utils/general.py | 14 ++++++++++---- tools/deployment/package/docker-compose.yaml | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 47d0472531..7c83983e5e 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -1,5 +1,6 @@ import enum import errno +import json import os import pathlib import re @@ -130,11 +131,16 @@ def generate_container_name(job_type: str) -> str: return f"clp-{job_type}-{str(uuid.uuid4())[-4:]}" -def is_docker_compose_running(): - cmd = ["docker", "compose", "ls", "--quiet"] +def is_docker_compose_running(project_name: str) -> bool: + cmd = [ + "docker", "compose", "ls", + "--format", "json", + "--filter", f"name={project_name}" + ] try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - return bool(output.strip()) + running_instances = json.loads(output) + return len(running_instances) >= 1 except subprocess.CalledProcessError: raise EnvironmentError("docker-compose is not installed or not functioning properly.") @@ -151,7 +157,7 @@ def check_docker_dependencies(should_compose_run: bool = False): except subprocess.CalledProcessError: raise EnvironmentError("docker is not installed or available on the path") - is_running = is_docker_compose_running() + is_running = is_docker_compose_running("clp-package") if should_compose_run and not is_running: raise EnvironmentError("docker-compose is not running.") if not should_compose_run and is_running: diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index fe42dd968f..ca6e6d7bf1 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -14,6 +14,8 @@ networks: clp-network: driver: "bridge" +name: clp-package + secrets: CLP_DB_PASS_FILE: environment: "CLP_DB_PASS" From 09ef298ed730fc1f8b4a0f3624eaab360eff9617 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 10 Sep 2025 13:30:28 -0400 Subject: [PATCH 066/238] fix lint --- components/clp-package-utils/clp_package_utils/general.py | 6 +----- tools/deployment/package/docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 7c83983e5e..f8679b6919 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -132,11 +132,7 @@ def generate_container_name(job_type: str) -> str: def is_docker_compose_running(project_name: str) -> bool: - cmd = [ - "docker", "compose", "ls", - "--format", "json", - "--filter", f"name={project_name}" - ] + cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) running_instances = json.loads(output) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index ca6e6d7bf1..ecd14a1799 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -14,7 +14,7 @@ networks: clp-network: driver: "bridge" -name: clp-package +name: "clp-package" secrets: CLP_DB_PASS_FILE: From d3b6a679f75667c79a889c92e8c68e768983842e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 10 Sep 2025 14:35:12 -0400 Subject: [PATCH 067/238] fix(taskfile): update default task to `docker-images:package` --- taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfile.yaml b/taskfile.yaml index 933b8012dc..984b6de348 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -48,7 +48,7 @@ vars: tasks: default: - deps: ["package"] + deps: ["docker-images:package"] clean: cmds: From 6148a65fdc001605bb77c5aa62442ba0f0807e2d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 17 Sep 2025 05:19:52 -0400 Subject: [PATCH 068/238] feat: reset default ports for container configs. --- .../clp-py-utils/clp_py_utils/clp_config.py | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 846b8a9b72..54548c9873 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,7 +1,7 @@ import os import pathlib from enum import auto -from typing import Literal, Optional, Union +from typing import ClassVar, Literal, Optional, Union from dotenv import dotenv_values from pydantic import BaseModel, PrivateAttr, root_validator, validator @@ -136,9 +136,11 @@ def validate_query_engine_package_compatibility(cls, values): class Database(BaseModel): + DEFAULT_PORT: ClassVar[int] = 3306 + type: str = "mariadb" host: str = "localhost" - port: int = 3306 + port: int = DEFAULT_PORT name: str = "clp-db" ssl_cert: Optional[str] = None auto_commit: bool = False @@ -240,6 +242,7 @@ def load_credentials_from_env(self): def transform_for_container_config(self): self.host = DB_COMPONENT_NAME + self.port = self.DEFAULT_PORT def _validate_logging_level(cls, field): @@ -275,6 +278,8 @@ def validate_logging_level(cls, field): class QueryScheduler(BaseModel): + DEFAULT_PORT: ClassVar[int] = 7000 + host: str = "localhost" port: int = 7000 jobs_poll_delay: float = 0.1 # seconds @@ -300,6 +305,7 @@ def validate_port(cls, field): def transform_for_container_config(self): self.host = QUERY_SCHEDULER_COMPONENT_NAME + self.port = self.DEFAULT_PORT class CompressionWorker(BaseModel): @@ -321,8 +327,10 @@ def validate_logging_level(cls, field): class Redis(BaseModel): + DEFAULT_PORT: ClassVar[int] = 6379 + host: str = "localhost" - port: int = 6379 + port: int = DEFAULT_PORT query_backend_database: int = 0 compression_backend_database: int = 1 # redis can perform authentication without a username @@ -356,11 +364,14 @@ def load_credentials_from_env(self): def transform_for_container_config(self): self.host = REDIS_COMPONENT_NAME + self.port = self.DEFAULT_PORT class Reducer(BaseModel): + DEFAULT_PORT: ClassVar[int] = 14009 + host: str = "localhost" - base_port: int = 14009 + base_port: int = DEFAULT_PORT logging_level: str = "INFO" upsert_interval: int = 100 # milliseconds @@ -389,11 +400,14 @@ def validate_upsert_interval(cls, field): def transform_for_container_config(self): self.host = REDUCER_COMPONENT_NAME + self.base_port = self.DEFAULT_PORT class ResultsCache(BaseModel): + DEFAULT_PORT: ClassVar[int] = 27017 + host: str = "localhost" - port: int = 27017 + port: int = DEFAULT_PORT db_name: str = "clp-query-results" stream_collection_name: str = "stream-files" retention_period: Optional[int] = 60 @@ -429,11 +443,14 @@ def get_uri(self): def transform_for_container_config(self): self.host = RESULTS_CACHE_COMPONENT_NAME + self.port = self.DEFAULT_PORT class Queue(BaseModel): + DEFAULT_PORT: ClassVar[int] = 5672 + host: str = "localhost" - port: int = 5672 + port: int = DEFAULT_PORT username: Optional[str] = None password: Optional[str] = None @@ -462,6 +479,7 @@ def load_credentials_from_env(self): def transform_for_container_config(self): self.host = QUEUE_COMPONENT_NAME + self.port = self.DEFAULT_PORT class S3Credentials(BaseModel): From 0ab99f0d0a525220f44c4229339d64c58d16b04d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 17 Sep 2025 05:26:41 -0400 Subject: [PATCH 069/238] fix(docker): update MongoDB connection string to use internal port --- tools/deployment/package/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index ecd14a1799..adba39dd42 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -129,7 +129,7 @@ services: <<: *healthcheck_defaults test: >- echo 'db.runCommand("ping").ok' | - mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet + mongosh 127.0.0.1:27017/test --quiet command: [ "--config", "/etc/mongo/mongod.conf", "--bind_ip", "0.0.0.0", From 263be6f600589377d5bf5360ec745bf243be2d48 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 17 Sep 2025 05:26:57 -0400 Subject: [PATCH 070/238] fix(controller): update webui configuration to use container-specific ports --- components/clp-package-utils/clp_package_utils/controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index d0f78f7bd9..49fbe1e6f9 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -298,11 +298,11 @@ def provision_webui(self, container_clp_config: CLPConfig): server_settings_json_updates = { "SqlDbHost": container_clp_config.database.host, - "SqlDbPort": self.clp_config.database.port, + "SqlDbPort": container_clp_config.database.port, "SqlDbName": self.clp_config.database.name, "SqlDbQueryJobsTableName": "query_jobs", "MongoDbHost": container_clp_config.results_cache.host, - "MongoDbPort": self.clp_config.results_cache.port, + "MongoDbPort": container_clp_config.results_cache.port, "MongoDbName": self.clp_config.results_cache.db_name, "MongoDbSearchResultsMetadataCollectionName": self.clp_config.webui.results_metadata_collection_name, "MongoDbStreamFilesCollectionName": self.clp_config.results_cache.stream_collection_name, From 51d1b55f5bca5fdd6704e0f62d443ea851662bc2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 17 Sep 2025 18:09:17 -0400 Subject: [PATCH 071/238] add ownership management for data and logs directories when running as root --- .../clp_package_utils/controller.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 49fbe1e6f9..eab3b46883 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -46,10 +46,40 @@ ) LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH +DEFAULT_CONTAINER_USER_ID = 999 +DEFAULT_CONTAINER_GROUP_ID = 999 logger = logging.getLogger(__name__) +def _chown_recursively( + path: pathlib.Path, + user_id: int, + group_id: int, +): + """ + Recursively changes the owner of the given path to the given user ID and group ID. + :param path: + :param user_id: + :param group_id: + """ + chown_cmd = ["chown", "--recursive", f"{user_id}:{group_id}", str(path)] + subprocess.run(chown_cmd, stdout=subprocess.DEVNULL, check=True) + + +def _chown_paths_if_root(*paths: pathlib.Path): + """ + Changes ownership of the given paths to the default container user/group IDs + if the current process is running as root. + + :param paths: + """ + if os.getuid() != 0: + return + for path in paths: + _chown_recursively(path, DEFAULT_CONTAINER_USER_ID, DEFAULT_CONTAINER_GROUP_ID) + + def _get_ip_from_hostname(hostname: str) -> str: """ Resolves a hostname to an IP address. @@ -76,6 +106,7 @@ def provision_database(self): validate_db_config(self.clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) + _chown_paths_if_root(data_dir, logs_dir) return { "CLP_DB_CONF_FILE_HOST": str(conf_file), @@ -98,6 +129,7 @@ def provision_queue(self): logs_dir = self.clp_config.logs_directory / component_name validate_queue_config(self.clp_config, logs_dir) logs_dir.mkdir(exist_ok=True, parents=True) + _chown_paths_if_root(logs_dir) return { "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), @@ -117,6 +149,7 @@ def provision_redis(self): validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) + _chown_paths_if_root(data_dir, logs_dir) return { "CLP_REDIS_CONF_FILE_HOST": str(conf_file), @@ -141,6 +174,7 @@ def provision_results_cache(self): validate_results_cache_config(self.clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) + _chown_paths_if_root(data_dir, logs_dir) return { "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), @@ -459,4 +493,4 @@ def _provision(self): with open(f"{self.clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): - env_file.write(f"{key}={value}\n") + env_file.write(f"{key}={value}\n") \ No newline at end of file From 0066386245481ba59afeadf27a77baea15b62469 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 17 Sep 2025 19:09:10 -0400 Subject: [PATCH 072/238] fix(controller, docker-compose): update user and group ID handling for containers --- .../clp_package_utils/controller.py | 12 +++++++----- tools/deployment/package/docker-compose.yaml | 6 +++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index eab3b46883..904a290d76 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -46,8 +46,10 @@ ) LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH -DEFAULT_CONTAINER_USER_ID = 999 -DEFAULT_CONTAINER_GROUP_ID = 999 +DEFAULT_UID_GID = f"{os.getuid()}:{os.getgid()}" +SERVICE_CONTAINER_USER_ID = 999 +SERVICE_CONTAINER_GROUP_ID = 999 +SERVICE_CONTAINER_UID_GID = f"{SERVICE_CONTAINER_USER_ID}:{SERVICE_CONTAINER_GROUP_ID}" logger = logging.getLogger(__name__) @@ -77,7 +79,7 @@ def _chown_paths_if_root(*paths: pathlib.Path): if os.getuid() != 0: return for path in paths: - _chown_recursively(path, DEFAULT_CONTAINER_USER_ID, DEFAULT_CONTAINER_GROUP_ID) + _chown_recursively(path, SERVICE_CONTAINER_USER_ID, SERVICE_CONTAINER_GROUP_ID) def _get_ip_from_hostname(hostname: str) -> str: @@ -463,8 +465,8 @@ def _provision(self): env_dict = { "CLP_PACKAGE_STORAGE_ENGINE": self.clp_config.package.storage_engine, # User and group IDs - "CLP_USER_ID": str(os.getuid()), - "CLP_GROUP_ID": str(os.getgid()), + "CLP_UID_GID": DEFAULT_UID_GID, + "CLP_SERVICE_CONTAINER_UID_GID": SERVICE_CONTAINER_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID, # Package container "CLP_PACKAGE_CONTAINER": self.clp_config.execution_container, # Global paths diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index adba39dd42..2e85855633 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -1,7 +1,7 @@ x-service-defaults: &service_defaults networks: ["clp-network"] stop_grace_period: "3s" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + user: "${CLP_UID_GID:-1000:1000}" x-healthcheck-defaults: &healthcheck_defaults interval: "30s" @@ -25,6 +25,7 @@ services: <<: *service_defaults container_name: "database" image: "${CLP_DB_IMAGE:-mysql:8.0.23}" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" environment: MYSQL_DATABASE: "${CLP_DB_NAME}" MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" @@ -74,6 +75,7 @@ services: <<: *service_defaults container_name: "queue" image: "rabbitmq:3.9.8" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" environment: RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" @@ -93,6 +95,7 @@ services: <<: *service_defaults container_name: "redis" image: "redis:7.2.4" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" volumes: @@ -119,6 +122,7 @@ services: <<: *service_defaults container_name: "results_cache" image: "mongo:7.0.1" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" volumes: From bc9ab99fce30375409a5d88e77b2a2a8afcf2c9b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Sep 2025 21:29:59 -0400 Subject: [PATCH 073/238] fix(search): enable direct connection for MongoDB client in search.py --- .../clp_package_utils/scripts/native/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/search.py b/components/clp-package-utils/clp_package_utils/scripts/native/search.py index d08d484463..60fd16d657 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/search.py @@ -76,7 +76,7 @@ def create_and_monitor_job_in_db( if do_count_aggregation is None and count_by_time_bucket_size is None: return - with pymongo.MongoClient(results_cache.get_uri()) as client: + with pymongo.MongoClient(results_cache.get_uri(), directConnection=True) as client: search_results_collection = client[results_cache.db_name][str(job_id)] if do_count_aggregation is not None: for document in search_results_collection.find(): From 80081385bd51d3f9576478edaf52f2ca8331fab0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 20 Sep 2025 03:53:42 -0400 Subject: [PATCH 074/238] revert direct connection option for MongoDB client in search.py --- .../clp_package_utils/scripts/native/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/search.py b/components/clp-package-utils/clp_package_utils/scripts/native/search.py index 60fd16d657..d08d484463 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/search.py @@ -76,7 +76,7 @@ def create_and_monitor_job_in_db( if do_count_aggregation is None and count_by_time_bucket_size is None: return - with pymongo.MongoClient(results_cache.get_uri(), directConnection=True) as client: + with pymongo.MongoClient(results_cache.get_uri()) as client: search_results_collection = client[results_cache.db_name][str(job_id)] if do_count_aggregation is not None: for document in search_results_collection.find(): From 34b60bd96ee340371c92eef46aae8edbd51e4350 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 01:38:13 -0400 Subject: [PATCH 075/238] refactor(controller): rename provision methods to set_up_env_for_* to better reflect their purpose. --- .../clp_package_utils/controller.py | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 904a290d76..2ad6fca226 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -98,9 +98,9 @@ def __init__(self, clp_config: CLPConfig): self.clp_home = get_clp_home() self._conf_dir = self.clp_home / "etc" - def provision_database(self): + def set_up_env_for_database(self): component_name = DB_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") conf_file = self._conf_dir / "mysql" / "conf.d" / "logging.cnf" data_dir = self.clp_config.data_directory / component_name @@ -124,9 +124,9 @@ def provision_database(self): ), } - def provision_queue(self): + def set_up_env_for_queue(self): component_name = QUEUE_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name validate_queue_config(self.clp_config, logs_dir) @@ -141,9 +141,9 @@ def provision_queue(self): "CLP_QUEUE_PASS": self.clp_config.queue.password, } - def provision_redis(self): + def set_up_env_for_redis(self): component_name = REDIS_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") conf_file = self._conf_dir / "redis" / "redis.conf" logs_dir = self.clp_config.logs_directory / component_name @@ -166,9 +166,9 @@ def provision_redis(self): ), } - def provision_results_cache(self): + def set_up_env_for_results_cache(self): component_name = RESULTS_CACHE_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") conf_file = self._conf_dir / "mongo" / "mongod.conf" data_dir = self.clp_config.data_directory / component_name @@ -188,9 +188,9 @@ def provision_results_cache(self): "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.clp_config.results_cache.stream_collection_name, } - def provision_compression_scheduler(self): + def set_up_env_for_compression_scheduler(self): component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_file = self.clp_config.logs_directory / f"{component_name}.log" logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) @@ -200,9 +200,9 @@ def provision_compression_scheduler(self): "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), } - def provision_query_scheduler(self): + def set_up_env_for_query_scheduler(self): component_name = QUERY_SCHEDULER_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_file = self.clp_config.logs_directory / f"{component_name}.log" logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) @@ -212,9 +212,9 @@ def provision_query_scheduler(self): "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), } - def provision_compression_worker(self, num_workers: int): + def set_up_env_for_compression_worker(self, num_workers: int): component_name = COMPRESSION_WORKER_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -225,9 +225,9 @@ def provision_compression_worker(self, num_workers: int): "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), } - def provision_query_worker(self, num_workers: int): + def set_up_env_for_query_worker(self, num_workers: int): component_name = QUERY_WORKER_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -238,9 +238,9 @@ def provision_query_worker(self, num_workers: int): "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } - def provision_reducer(self, num_workers: int): + def set_up_env_for_reducer(self, num_workers: int): component_name = REDUCER_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -292,9 +292,9 @@ def _read_and_update_settings_json( return settings_object - def provision_webui(self, container_clp_config: CLPConfig): + def set_up_env_for_webui(self, container_clp_config: CLPConfig): component_name = WEBUI_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" client_settings_json_path = ( @@ -380,9 +380,9 @@ def provision_webui(self, container_clp_config: CLPConfig): "CLP_WEBUI_RATE_LIMIT": str(self.clp_config.webui.rate_limit), } - def provision_garbage_collector(self): + def set_up_env_for_garbage_collector(self): component_name = GARBAGE_COLLECTOR_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -477,17 +477,17 @@ def _provision(self): # AWS credentials "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - **self.provision_database(), - **self.provision_queue(), - **self.provision_redis(), - **self.provision_results_cache(), - **self.provision_compression_scheduler(), - **self.provision_query_scheduler(), - **self.provision_compression_worker(num_workers), - **self.provision_query_worker(num_workers), - **self.provision_reducer(num_workers), - **self.provision_webui(container_clp_config), - **self.provision_garbage_collector(), + **self.set_up_env_for_database(), + **self.set_up_env_for_queue(), + **self.set_up_env_for_redis(), + **self.set_up_env_for_results_cache(), + **self.set_up_env_for_compression_scheduler(), + **self.set_up_env_for_query_scheduler(), + **self.set_up_env_for_compression_worker(num_workers), + **self.set_up_env_for_query_worker(num_workers), + **self.set_up_env_for_reducer(num_workers), + **self.set_up_env_for_webui(container_clp_config), + **self.set_up_env_for_garbage_collector(), } if self.clp_config.aws_config_directory is not None: From 9f6348120f334fc265a588c1576adbcbf29cd3eb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 01:44:05 -0400 Subject: [PATCH 076/238] reorder private functions --- .../clp_package_utils/controller.py | 174 +++++++++--------- 1 file changed, 87 insertions(+), 87 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 2ad6fca226..ca9fa0f30e 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -54,44 +54,6 @@ logger = logging.getLogger(__name__) -def _chown_recursively( - path: pathlib.Path, - user_id: int, - group_id: int, -): - """ - Recursively changes the owner of the given path to the given user ID and group ID. - :param path: - :param user_id: - :param group_id: - """ - chown_cmd = ["chown", "--recursive", f"{user_id}:{group_id}", str(path)] - subprocess.run(chown_cmd, stdout=subprocess.DEVNULL, check=True) - - -def _chown_paths_if_root(*paths: pathlib.Path): - """ - Changes ownership of the given paths to the default container user/group IDs - if the current process is running as root. - - :param paths: - """ - if os.getuid() != 0: - return - for path in paths: - _chown_recursively(path, SERVICE_CONTAINER_USER_ID, SERVICE_CONTAINER_GROUP_ID) - - -def _get_ip_from_hostname(hostname: str) -> str: - """ - Resolves a hostname to an IP address. - - :param hostname: The hostname to resolve. - :return: The resolved IP address. - """ - return socket.gethostbyname(hostname) - - class BaseController(ABC): def __init__(self, clp_config: CLPConfig): self.clp_config = clp_config @@ -252,46 +214,6 @@ def set_up_env_for_reducer(self, num_workers: int): "CLP_REDUCER_UPSERT_INTERVAL": str(self.clp_config.reducer.upsert_interval), } - def _update_settings_object( - self, - parent_key_prefix: str, - settings: Dict[str, Any], - updates: Dict[str, Any], - ): - """ - Recursively updates the given settings object with the values from `updates`. - - :param parent_key_prefix: The prefix for keys at this level in the settings dictionary. - :param settings: The settings to update. - :param updates: The updates. - :raises ValueError: If a key in `updates` doesn't exist in `settings`. - """ - for key, value in updates.items(): - if key not in settings: - error_msg = ( - f"{parent_key_prefix}{key} is not a valid configuration key for the webui." - ) - raise ValueError(error_msg) - if isinstance(value, dict): - self._update_settings_object(f"{parent_key_prefix}{key}.", settings[key], value) - else: - settings[key] = updates[key] - - def _read_and_update_settings_json( - self, settings_file_path: pathlib.Path, updates: Dict[str, Any] - ): - """ - Reads and updates a settings JSON file. - - :param settings_file_path: - :param updates: - """ - with open(settings_file_path, "r") as settings_json_file: - settings_object = json.loads(settings_json_file.read()) - self._update_settings_object("", settings_object, updates) - - return settings_object - def set_up_env_for_webui(self, container_clp_config: CLPConfig): component_name = WEBUI_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -403,6 +325,46 @@ def stop(self): """ pass + def _update_settings_object( + self, + parent_key_prefix: str, + settings: Dict[str, Any], + updates: Dict[str, Any], + ): + """ + Recursively updates the given settings object with the values from `updates`. + + :param parent_key_prefix: The prefix for keys at this level in the settings dictionary. + :param settings: The settings to update. + :param updates: The updates. + :raises ValueError: If a key in `updates` doesn't exist in `settings`. + """ + for key, value in updates.items(): + if key not in settings: + error_msg = ( + f"{parent_key_prefix}{key} is not a valid configuration key for the webui." + ) + raise ValueError(error_msg) + if isinstance(value, dict): + self._update_settings_object(f"{parent_key_prefix}{key}.", settings[key], value) + else: + settings[key] = updates[key] + + def _read_and_update_settings_json( + self, settings_file_path: pathlib.Path, updates: Dict[str, Any] + ): + """ + Reads and updates a settings JSON file. + + :param settings_file_path: + :param updates: + """ + with open(settings_file_path, "r") as settings_json_file: + settings_object = json.loads(settings_json_file.read()) + self._update_settings_object("", settings_object, updates) + + return settings_object + @abstractmethod def _provision(self) -> Dict[str, str]: """ @@ -414,14 +376,6 @@ def _provision(self) -> Dict[str, str]: class DockerComposeController(BaseController): - @staticmethod - def _get_num_workers(): - """ - Gets the parallelism number for worker components. - TODO: Revisit after moving from single-container to multi-container workers. - """ - return multiprocessing.cpu_count() // 2 - def __init__(self, clp_config: CLPConfig): super().__init__(clp_config) @@ -457,6 +411,14 @@ def stop(self): logger.exception("Failed to stop CLP containers using Docker Compose.") raise + @staticmethod + def _get_num_workers(): + """ + Gets the parallelism number for worker components. + TODO: Revisit after moving from single-container to multi-container workers. + """ + return multiprocessing.cpu_count() // 2 + def _provision(self): container_clp_config = generate_docker_compose_container_config(self.clp_config) num_workers = self._get_num_workers() @@ -495,4 +457,42 @@ def _provision(self): with open(f"{self.clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): - env_file.write(f"{key}={value}\n") \ No newline at end of file + env_file.write(f"{key}={value}\n") + + +def _chown_recursively( + path: pathlib.Path, + user_id: int, + group_id: int, +): + """ + Recursively changes the owner of the given path to the given user ID and group ID. + :param path: + :param user_id: + :param group_id: + """ + chown_cmd = ["chown", "--recursive", f"{user_id}:{group_id}", str(path)] + subprocess.run(chown_cmd, stdout=subprocess.DEVNULL, check=True) + + +def _chown_paths_if_root(*paths: pathlib.Path): + """ + Changes ownership of the given paths to the default container user/group IDs + if the current process is running as root. + + :param paths: + """ + if os.getuid() != 0: + return + for path in paths: + _chown_recursively(path, SERVICE_CONTAINER_USER_ID, SERVICE_CONTAINER_GROUP_ID) + + +def _get_ip_from_hostname(hostname: str) -> str: + """ + Resolves a hostname to an IP address. + + :param hostname: The hostname to resolve. + :return: The resolved IP address. + """ + return socket.gethostbyname(hostname) \ No newline at end of file From 2344db9007a7bdd215a54d58530557a58a451e1e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 02:44:47 -0400 Subject: [PATCH 077/238] add documentation --- .../clp_package_utils/controller.py | 96 ++++++++++++++++++- 1 file changed, 92 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index ca9fa0f30e..22a7be9f05 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -55,12 +55,23 @@ class BaseController(ABC): + """ + Abstract base controller for preparing and deploying CLP components. + Provides common logic for preparing environment variables, directories, + and configuration files for each service. + """ + def __init__(self, clp_config: CLPConfig): self.clp_config = clp_config self.clp_home = get_clp_home() self._conf_dir = self.clp_home / "etc" def set_up_env_for_database(self): + """ + Prepares environment variables and directories for the database component. + + :return: Dictionary of component-related environment variables. + """ component_name = DB_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -87,6 +98,11 @@ def set_up_env_for_database(self): } def set_up_env_for_queue(self): + """ + Prepares environment variables and directories for the message queue component. + + :return: Dictionary of component-related environment variables. + """ component_name = QUEUE_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -104,6 +120,11 @@ def set_up_env_for_queue(self): } def set_up_env_for_redis(self): + """ + Prepares environment variables and directories for the Redis component. + + :return: Dictionary of component-related environment variables. + """ component_name = REDIS_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -129,6 +150,11 @@ def set_up_env_for_redis(self): } def set_up_env_for_results_cache(self): + """ + Prepares environment variables and directories for the results cache (MongoDB) component. + + :return: Dictionary of component-related environment variables. + """ component_name = RESULTS_CACHE_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -151,6 +177,11 @@ def set_up_env_for_results_cache(self): } def set_up_env_for_compression_scheduler(self): + """ + Prepares environment variables and files for the compression scheduler component. + + :return: Dictionary of component-related environment variables. + """ component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -163,6 +194,11 @@ def set_up_env_for_compression_scheduler(self): } def set_up_env_for_query_scheduler(self): + """ + Prepares environment variables and files for the query scheduler component. + + :return: Dictionary of component-related environment variables. + """ component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -175,6 +211,12 @@ def set_up_env_for_query_scheduler(self): } def set_up_env_for_compression_worker(self, num_workers: int): + """ + Prepares environment variables for the compression worker component. + + :param num_workers: Number of worker processes to run. + :return: Dictionary of compression worker-related environment variables. + """ component_name = COMPRESSION_WORKER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -188,6 +230,12 @@ def set_up_env_for_compression_worker(self, num_workers: int): } def set_up_env_for_query_worker(self, num_workers: int): + """ + Prepares environment variables for the query worker component. + + :param num_workers: Number of worker processes to run. + :return: Dictionary of component-related environment variables. + """ component_name = QUERY_WORKER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -201,6 +249,12 @@ def set_up_env_for_query_worker(self, num_workers: int): } def set_up_env_for_reducer(self, num_workers: int): + """ + Prepares environment variables for the reducer component. + + :param num_workers: Number of worker processes to run. + :return: Dictionary of component-related environment variables. + """ component_name = REDUCER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -215,6 +269,12 @@ def set_up_env_for_reducer(self, num_workers: int): } def set_up_env_for_webui(self, container_clp_config: CLPConfig): + """ + Prepares environment variables and settings for the Web UI component. + + :param container_clp_config: CLP configuration inside the containers. + :return: Dictionary of component-related environment variables. + """ component_name = WEBUI_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -303,6 +363,11 @@ def set_up_env_for_webui(self, container_clp_config: CLPConfig): } def set_up_env_for_garbage_collector(self): + """ + Prepares environment variables for the garbage collector component. + + :return: Dictionary of component-related environment variables. + """ component_name = GARBAGE_COLLECTOR_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -368,18 +433,28 @@ def _read_and_update_settings_json( @abstractmethod def _provision(self) -> Dict[str, str]: """ - Provisions all components with orchestrator-specific logic. + Prepares all components with orchestrator-specific logic. - :return: Dictionary of environment variables for the orchestrator + :return: Dictionary of environment variables to be used by the orchestrator. """ pass class DockerComposeController(BaseController): + """ + Controller for deploying CLP components using Docker Compose. + """ + def __init__(self, clp_config: CLPConfig): super().__init__(clp_config) def deploy(self): + """ + Deploys CLP components using Docker Compose by: + 1. Checking Docker dependencies. + 2. Provisioning environment variables and configuration. + 3. Running `docker compose up -d`. + """ check_docker_dependencies(should_compose_run=False) self._provision() @@ -396,6 +471,9 @@ def deploy(self): raise def stop(self): + """ + Stops CLP components deployed via Docker Compose. + """ check_docker_dependencies(should_compose_run=True) logger.info("Stopping all CLP containers using Docker Compose...") @@ -416,10 +494,19 @@ def _get_num_workers(): """ Gets the parallelism number for worker components. TODO: Revisit after moving from single-container to multi-container workers. + :return: Number of worker processes. """ return multiprocessing.cpu_count() // 2 def _provision(self): + """ + Provisions all CLP components for Docker Compose by: + - Generating container-specific config. + - Preparing environment variables for all components. + - Writing environment variables to `.env`. + + :return: Dictionary of all environment variables. + """ container_clp_config = generate_docker_compose_container_config(self.clp_config) num_workers = self._get_num_workers() dump_shared_container_config(container_clp_config, self.clp_config) @@ -467,6 +554,7 @@ def _chown_recursively( ): """ Recursively changes the owner of the given path to the given user ID and group ID. + :param path: :param user_id: :param group_id: @@ -477,8 +565,8 @@ def _chown_recursively( def _chown_paths_if_root(*paths: pathlib.Path): """ - Changes ownership of the given paths to the default container user/group IDs - if the current process is running as root. + Changes ownership of the given paths to the default service container user/group IDs if the + current process is running as root. :param paths: """ From 2ba93d7d212b8c001b64e4cd6338d63a5f806e40 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 02:53:05 -0400 Subject: [PATCH 078/238] change visibility of `set_up_env_for` function from public to private by prefixing `_`; reorder methods. --- .../clp_package_utils/controller.py | 144 +++++++++--------- 1 file changed, 73 insertions(+), 71 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 22a7be9f05..340bed1763 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -66,7 +66,30 @@ def __init__(self, clp_config: CLPConfig): self.clp_home = get_clp_home() self._conf_dir = self.clp_home / "etc" - def set_up_env_for_database(self): + @abstractmethod + def deploy(self): + """ + Deploys the provisioned components with orchestrator-specific logic. + """ + pass + + @abstractmethod + def stop(self): + """ + Stops the deployed components with orchestrator-specific logic. + """ + pass + + @abstractmethod + def _provision(self) -> Dict[str, str]: + """ + Prepares all components with orchestrator-specific logic. + + :return: Dictionary of environment variables to be used by the orchestrator. + """ + pass + + def _set_up_env_for_database(self): """ Prepares environment variables and directories for the database component. @@ -97,7 +120,7 @@ def set_up_env_for_database(self): ), } - def set_up_env_for_queue(self): + def _set_up_env_for_queue(self): """ Prepares environment variables and directories for the message queue component. @@ -119,7 +142,7 @@ def set_up_env_for_queue(self): "CLP_QUEUE_PASS": self.clp_config.queue.password, } - def set_up_env_for_redis(self): + def _set_up_env_for_redis(self): """ Prepares environment variables and directories for the Redis component. @@ -149,7 +172,7 @@ def set_up_env_for_redis(self): ), } - def set_up_env_for_results_cache(self): + def _set_up_env_for_results_cache(self): """ Prepares environment variables and directories for the results cache (MongoDB) component. @@ -176,7 +199,7 @@ def set_up_env_for_results_cache(self): "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.clp_config.results_cache.stream_collection_name, } - def set_up_env_for_compression_scheduler(self): + def _set_up_env_for_compression_scheduler(self): """ Prepares environment variables and files for the compression scheduler component. @@ -193,7 +216,7 @@ def set_up_env_for_compression_scheduler(self): "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), } - def set_up_env_for_query_scheduler(self): + def _set_up_env_for_query_scheduler(self): """ Prepares environment variables and files for the query scheduler component. @@ -210,7 +233,7 @@ def set_up_env_for_query_scheduler(self): "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), } - def set_up_env_for_compression_worker(self, num_workers: int): + def _set_up_env_for_compression_worker(self, num_workers: int): """ Prepares environment variables for the compression worker component. @@ -229,7 +252,7 @@ def set_up_env_for_compression_worker(self, num_workers: int): "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), } - def set_up_env_for_query_worker(self, num_workers: int): + def _set_up_env_for_query_worker(self, num_workers: int): """ Prepares environment variables for the query worker component. @@ -248,7 +271,7 @@ def set_up_env_for_query_worker(self, num_workers: int): "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } - def set_up_env_for_reducer(self, num_workers: int): + def _set_up_env_for_reducer(self, num_workers: int): """ Prepares environment variables for the reducer component. @@ -268,7 +291,7 @@ def set_up_env_for_reducer(self, num_workers: int): "CLP_REDUCER_UPSERT_INTERVAL": str(self.clp_config.reducer.upsert_interval), } - def set_up_env_for_webui(self, container_clp_config: CLPConfig): + def _set_up_env_for_webui(self, container_clp_config: CLPConfig): """ Prepares environment variables and settings for the Web UI component. @@ -362,7 +385,7 @@ def set_up_env_for_webui(self, container_clp_config: CLPConfig): "CLP_WEBUI_RATE_LIMIT": str(self.clp_config.webui.rate_limit), } - def set_up_env_for_garbage_collector(self): + def _set_up_env_for_garbage_collector(self): """ Prepares environment variables for the garbage collector component. @@ -376,19 +399,20 @@ def set_up_env_for_garbage_collector(self): return {"CLP_GC_LOGGING_LEVEL": self.clp_config.garbage_collector.logging_level} - @abstractmethod - def deploy(self): - """ - Deploys the provisioned components with orchestrator-specific logic. + def _read_and_update_settings_json( + self, settings_file_path: pathlib.Path, updates: Dict[str, Any] + ): """ - pass + Reads and updates a settings JSON file. - @abstractmethod - def stop(self): - """ - Stops the deployed components with orchestrator-specific logic. + :param settings_file_path: + :param updates: """ - pass + with open(settings_file_path, "r") as settings_json_file: + settings_object = json.loads(settings_json_file.read()) + self._update_settings_object("", settings_object, updates) + + return settings_object def _update_settings_object( self, @@ -415,30 +439,6 @@ def _update_settings_object( else: settings[key] = updates[key] - def _read_and_update_settings_json( - self, settings_file_path: pathlib.Path, updates: Dict[str, Any] - ): - """ - Reads and updates a settings JSON file. - - :param settings_file_path: - :param updates: - """ - with open(settings_file_path, "r") as settings_json_file: - settings_object = json.loads(settings_json_file.read()) - self._update_settings_object("", settings_object, updates) - - return settings_object - - @abstractmethod - def _provision(self) -> Dict[str, str]: - """ - Prepares all components with orchestrator-specific logic. - - :return: Dictionary of environment variables to be used by the orchestrator. - """ - pass - class DockerComposeController(BaseController): """ @@ -515,7 +515,9 @@ def _provision(self): "CLP_PACKAGE_STORAGE_ENGINE": self.clp_config.package.storage_engine, # User and group IDs "CLP_UID_GID": DEFAULT_UID_GID, - "CLP_SERVICE_CONTAINER_UID_GID": SERVICE_CONTAINER_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID, + "CLP_SERVICE_CONTAINER_UID_GID": ( + SERVICE_CONTAINER_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID + ), # Package container "CLP_PACKAGE_CONTAINER": self.clp_config.execution_container, # Global paths @@ -526,17 +528,17 @@ def _provision(self): # AWS credentials "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - **self.set_up_env_for_database(), - **self.set_up_env_for_queue(), - **self.set_up_env_for_redis(), - **self.set_up_env_for_results_cache(), - **self.set_up_env_for_compression_scheduler(), - **self.set_up_env_for_query_scheduler(), - **self.set_up_env_for_compression_worker(num_workers), - **self.set_up_env_for_query_worker(num_workers), - **self.set_up_env_for_reducer(num_workers), - **self.set_up_env_for_webui(container_clp_config), - **self.set_up_env_for_garbage_collector(), + **self._set_up_env_for_database(), + **self._set_up_env_for_queue(), + **self._set_up_env_for_redis(), + **self._set_up_env_for_results_cache(), + **self._set_up_env_for_compression_scheduler(), + **self._set_up_env_for_query_scheduler(), + **self._set_up_env_for_compression_worker(num_workers), + **self._set_up_env_for_query_worker(num_workers), + **self._set_up_env_for_reducer(num_workers), + **self._set_up_env_for_webui(container_clp_config), + **self._set_up_env_for_garbage_collector(), } if self.clp_config.aws_config_directory is not None: @@ -547,6 +549,19 @@ def _provision(self): env_file.write(f"{key}={value}\n") +def _chown_paths_if_root(*paths: pathlib.Path): + """ + Changes ownership of the given paths to the default service container user/group IDs if the + current process is running as root. + + :param paths: + """ + if os.getuid() != 0: + return + for path in paths: + _chown_recursively(path, SERVICE_CONTAINER_USER_ID, SERVICE_CONTAINER_GROUP_ID) + + def _chown_recursively( path: pathlib.Path, user_id: int, @@ -563,19 +578,6 @@ def _chown_recursively( subprocess.run(chown_cmd, stdout=subprocess.DEVNULL, check=True) -def _chown_paths_if_root(*paths: pathlib.Path): - """ - Changes ownership of the given paths to the default service container user/group IDs if the - current process is running as root. - - :param paths: - """ - if os.getuid() != 0: - return - for path in paths: - _chown_recursively(path, SERVICE_CONTAINER_USER_ID, SERVICE_CONTAINER_GROUP_ID) - - def _get_ip_from_hostname(hostname: str) -> str: """ Resolves a hostname to an IP address. @@ -583,4 +585,4 @@ def _get_ip_from_hostname(hostname: str) -> str: :param hostname: The hostname to resolve. :return: The resolved IP address. """ - return socket.gethostbyname(hostname) \ No newline at end of file + return socket.gethostbyname(hostname) From ce94225d8a356451b196c74fc4b370cd2ccfa751 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:05:08 -0400 Subject: [PATCH 079/238] Improve type hints. --- .../clp_package_utils/controller.py | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 340bed1763..2251d5d770 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -9,6 +9,9 @@ from abc import ABC, abstractmethod from typing import Any, Dict +# Type alias for environment variables dictionary. +EnvVarsDict = Dict[str, str] + from clp_py_utils.clp_config import ( AwsAuthType, CLPConfig, @@ -81,7 +84,7 @@ def stop(self): pass @abstractmethod - def _provision(self) -> Dict[str, str]: + def _provision(self) -> EnvVarsDict: """ Prepares all components with orchestrator-specific logic. @@ -89,7 +92,7 @@ def _provision(self) -> Dict[str, str]: """ pass - def _set_up_env_for_database(self): + def _set_up_env_for_database(self) -> EnvVarsDict: """ Prepares environment variables and directories for the database component. @@ -120,7 +123,7 @@ def _set_up_env_for_database(self): ), } - def _set_up_env_for_queue(self): + def _set_up_env_for_queue(self) -> EnvVarsDict: """ Prepares environment variables and directories for the message queue component. @@ -142,7 +145,7 @@ def _set_up_env_for_queue(self): "CLP_QUEUE_PASS": self.clp_config.queue.password, } - def _set_up_env_for_redis(self): + def _set_up_env_for_redis(self) -> EnvVarsDict: """ Prepares environment variables and directories for the Redis component. @@ -172,7 +175,7 @@ def _set_up_env_for_redis(self): ), } - def _set_up_env_for_results_cache(self): + def _set_up_env_for_results_cache(self) -> EnvVarsDict: """ Prepares environment variables and directories for the results cache (MongoDB) component. @@ -199,7 +202,7 @@ def _set_up_env_for_results_cache(self): "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.clp_config.results_cache.stream_collection_name, } - def _set_up_env_for_compression_scheduler(self): + def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: """ Prepares environment variables and files for the compression scheduler component. @@ -216,7 +219,7 @@ def _set_up_env_for_compression_scheduler(self): "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), } - def _set_up_env_for_query_scheduler(self): + def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: """ Prepares environment variables and files for the query scheduler component. @@ -233,7 +236,7 @@ def _set_up_env_for_query_scheduler(self): "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), } - def _set_up_env_for_compression_worker(self, num_workers: int): + def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: """ Prepares environment variables for the compression worker component. @@ -252,7 +255,7 @@ def _set_up_env_for_compression_worker(self, num_workers: int): "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), } - def _set_up_env_for_query_worker(self, num_workers: int): + def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: """ Prepares environment variables for the query worker component. @@ -271,7 +274,7 @@ def _set_up_env_for_query_worker(self, num_workers: int): "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } - def _set_up_env_for_reducer(self, num_workers: int): + def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: """ Prepares environment variables for the reducer component. @@ -291,7 +294,7 @@ def _set_up_env_for_reducer(self, num_workers: int): "CLP_REDUCER_UPSERT_INTERVAL": str(self.clp_config.reducer.upsert_interval), } - def _set_up_env_for_webui(self, container_clp_config: CLPConfig): + def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: """ Prepares environment variables and settings for the Web UI component. @@ -385,7 +388,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig): "CLP_WEBUI_RATE_LIMIT": str(self.clp_config.webui.rate_limit), } - def _set_up_env_for_garbage_collector(self): + def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: """ Prepares environment variables for the garbage collector component. @@ -401,7 +404,7 @@ def _set_up_env_for_garbage_collector(self): def _read_and_update_settings_json( self, settings_file_path: pathlib.Path, updates: Dict[str, Any] - ): + ) -> Dict[str, Any]: """ Reads and updates a settings JSON file. @@ -490,7 +493,7 @@ def stop(self): raise @staticmethod - def _get_num_workers(): + def _get_num_workers() -> int: """ Gets the parallelism number for worker components. TODO: Revisit after moving from single-container to multi-container workers. From 52258705962e42ef406d3be97deca8baace0d1c1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:09:05 -0400 Subject: [PATCH 080/238] docs(clp-package-utils): add docstring for is_docker_compose_running function. --- components/clp-package-utils/clp_package_utils/general.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index f8679b6919..bdd3b38bb8 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -132,6 +132,13 @@ def generate_container_name(job_type: str) -> str: def is_docker_compose_running(project_name: str) -> bool: + """ + Checks if a Docker Compose project is running. + + :param project_name: + :return: True if at least one instance is running, else False. + :raises EnvironmentError: If Docker Compose is not installed or fails. + """ cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) From cecf5b28c010c8f4c6ca291ae4601cda972666dc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:11:31 -0400 Subject: [PATCH 081/238] docs(clp-package-utils): add docstring for check_docker_dependencies function. --- components/clp-package-utils/clp_package_utils/general.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index bdd3b38bb8..cfb03e9ffd 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -149,6 +149,13 @@ def is_docker_compose_running(project_name: str) -> bool: def check_docker_dependencies(should_compose_run: bool = False): + """ + Checks if Docker and Docker Compose are installed, and whether Docker Compose is running or not. + + :param should_compose_run: + :raises EnvironmentError: If any Docker dependency is not installed or Docker Compose state + does not match expectation. + """ try: subprocess.run( "command -v docker", From 0c0c562be5b232c26327bf0a80d6b4c9d5a2c985 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:12:57 -0400 Subject: [PATCH 082/238] docs(clp-package-utils): add docstring for _validate_log_directory function. --- components/clp-package-utils/clp_package_utils/general.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index cfb03e9ffd..b2bba35be7 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -175,6 +175,13 @@ def check_docker_dependencies(should_compose_run: bool = False): def _validate_log_directory(logs_dir: pathlib.Path, component_name: str) -> None: + """ + Validate that a log directory path of a component is valid. + + :param logs_dir: + :param component_name: + :raises ValueError: If the path is invalid or not a directory. + """ try: validate_path_could_be_dir(logs_dir) except ValueError as ex: From 762884b5bf3b6e403b3ecbfe6f1ee03a81359fb3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:13:13 -0400 Subject: [PATCH 083/238] remove `None` return type annotation from _validate_log_directory function. --- components/clp-package-utils/clp_package_utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index b2bba35be7..fcc3a3dd77 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -174,7 +174,7 @@ def check_docker_dependencies(should_compose_run: bool = False): raise EnvironmentError("docker-compose is already running.") -def _validate_log_directory(logs_dir: pathlib.Path, component_name: str) -> None: +def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): """ Validate that a log directory path of a component is valid. From fc515dbcd580e4bf8d9e59daec09773029bff104 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:19:02 -0400 Subject: [PATCH 084/238] Rename transform_for_container_config to transform_for_container; add docstring for CLPConfig.transform_for_container --- .../clp_package_utils/general.py | 2 +- .../clp-py-utils/clp_py_utils/clp_config.py | 52 +++++++++++-------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index fcc3a3dd77..fd357aad36 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -314,7 +314,7 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig :return: The container config and the mounts. """ container_clp_config = clp_config.copy(deep=True) - container_clp_config.transform_for_container_config() + container_clp_config.transform_for_container() return container_clp_config diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 54548c9873..30962ca83a 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -240,7 +240,7 @@ def load_credentials_from_env(self): self.username = _get_env_var(CLP_DB_USER_ENV_VAR_NAME) self.password = _get_env_var(CLP_DB_PASS_ENV_VAR_NAME) - def transform_for_container_config(self): + def transform_for_container(self): self.host = DB_COMPONENT_NAME self.port = self.DEFAULT_PORT @@ -303,7 +303,7 @@ def validate_port(cls, field): raise ValueError(f"{field} is not greater than zero") return field - def transform_for_container_config(self): + def transform_for_container(self): self.host = QUERY_SCHEDULER_COMPONENT_NAME self.port = self.DEFAULT_PORT @@ -362,7 +362,7 @@ def load_credentials_from_env(self): """ self.password = _get_env_var(CLP_REDIS_PASS_ENV_VAR_NAME) - def transform_for_container_config(self): + def transform_for_container(self): self.host = REDIS_COMPONENT_NAME self.port = self.DEFAULT_PORT @@ -398,7 +398,7 @@ def validate_upsert_interval(cls, field): raise ValueError(f"{field} is not greater than zero") return field - def transform_for_container_config(self): + def transform_for_container(self): self.host = REDUCER_COMPONENT_NAME self.base_port = self.DEFAULT_PORT @@ -441,7 +441,7 @@ def validate_retention_period(cls, field): def get_uri(self): return f"mongodb://{self.host}:{self.port}/{self.db_name}" - def transform_for_container_config(self): + def transform_for_container(self): self.host = RESULTS_CACHE_COMPONENT_NAME self.port = self.DEFAULT_PORT @@ -477,7 +477,7 @@ def load_credentials_from_env(self): self.username = _get_env_var(CLP_QUEUE_USER_ENV_VAR_NAME) self.password = _get_env_var(CLP_QUEUE_PASS_ENV_VAR_NAME) - def transform_for_container_config(self): + def transform_for_container(self): self.host = QUEUE_COMPONENT_NAME self.port = self.DEFAULT_PORT @@ -558,7 +558,7 @@ class S3IngestionConfig(BaseModel): def dump_to_primitive_dict(self): return self.dict() - def transform_for_container_config(self): + def transform_for_container(self): pass @@ -616,35 +616,35 @@ def dump_to_primitive_dict(self): class FsIngestionConfig(FsStorage): directory: pathlib.Path = pathlib.Path("/") - def transform_for_container_config(self): + def transform_for_container(self): self.directory = CONTAINER_INPUT_LOGS_ROOT_DIR class ArchiveFsStorage(FsStorage): directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH - def transform_for_container_config(self): + def transform_for_container(self): self.directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH class StreamFsStorage(FsStorage): directory: pathlib.Path = CLP_DEFAULT_STREAM_DIRECTORY_PATH - def transform_for_container_config(self): + def transform_for_container(self): self.directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH class ArchiveS3Storage(S3Storage): staging_directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH - def transform_for_container_config(self): + def transform_for_container(self): self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH class StreamS3Storage(S3Storage): staging_directory: pathlib.Path = CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH - def transform_for_container_config(self): + def transform_for_container(self): self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH @@ -970,21 +970,27 @@ def dump_to_primitive_dict(self): return d - def transform_for_container_config(self): + def transform_for_container(self): + """ + Adjusts paths and service hosts for containerized execution. + + Converts all relevant directories to absolute paths inside the container + and updates service hostnames/ports to their container service names. + """ self.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH self.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH if self.aws_config_directory is not None: self.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY - self.logs_input.transform_for_container_config() - self.archive_output.storage.transform_for_container_config() - self.stream_output.storage.transform_for_container_config() - - self.database.transform_for_container_config() - self.queue.transform_for_container_config() - self.redis.transform_for_container_config() - self.results_cache.transform_for_container_config() - self.query_scheduler.transform_for_container_config() - self.reducer.transform_for_container_config() + self.logs_input.transform_for_container() + self.archive_output.storage.transform_for_container() + self.stream_output.storage.transform_for_container() + + self.database.transform_for_container() + self.queue.transform_for_container() + self.redis.transform_for_container() + self.results_cache.transform_for_container() + self.query_scheduler.transform_for_container() + self.reducer.transform_for_container() class WorkerConfig(BaseModel): From 268c1444cac04408d618ae5335a68f416f521887 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:22:20 -0400 Subject: [PATCH 085/238] add docs for x-service-defaults & x-healthcheck-defaults in docker-compose.yaml --- tools/deployment/package/docker-compose.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 2e85855633..0f9a7243d5 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -1,8 +1,10 @@ +# Common service defaults. x-service-defaults: &service_defaults networks: ["clp-network"] stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" +# Common healthcheck defaults. x-healthcheck-defaults: &healthcheck_defaults interval: "30s" retries: 3 From fc7aae361433f3cf59fe19cfdeb03d3bf0cdad04 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 24 Sep 2025 15:59:03 -0400 Subject: [PATCH 086/238] remove garbage-collector's dependency condition on query-scheduler. --- docs/src/dev-docs/design-docker-compose.md | 1 - tools/deployment/package/docker-compose.yaml | 2 -- 2 files changed, 3 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index d845a242ae..d8fa7800ad 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -82,7 +82,6 @@ graph LR db_table_creator -->|completed_successfully| webui results_cache_indices_creator -->|completed_successfully| webui db_table_creator -->|completed_successfully| garbage_collector - query_scheduler -->|healthy| garbage_collector results_cache_indices_creator -->|completed_successfully| garbage_collector subgraph Databases diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 0f9a7243d5..64aa7232eb 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -385,8 +385,6 @@ services: depends_on: db-table-creator: condition: "service_completed_successfully" - query-scheduler: - condition: "service_healthy" results-cache-indices-creator: condition: "service_completed_successfully" command: [ From cda03486b4284efb2e64c2007312b559282fa63f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 24 Sep 2025 19:00:51 -0400 Subject: [PATCH 087/238] Split base services into a separate docker-compose.base.yaml; Launch only base services when Presto query engine is configured. --- .../clp_package_utils/controller.py | 11 +- .../clp-py-utils/clp_py_utils/clp_config.py | 50 +-- taskfile.yaml | 4 +- .../package/docker-compose.base.yaml | 293 ++++++++++++++++++ tools/deployment/package/docker-compose.yaml | 284 +---------------- 5 files changed, 319 insertions(+), 323 deletions(-) create mode 100644 tools/deployment/package/docker-compose.base.yaml diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index a4b763915d..ef9a1870f8 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -28,7 +28,7 @@ RESULTS_CACHE_COMPONENT_NAME, StorageEngine, StorageType, - WEBUI_COMPONENT_NAME, + WEBUI_COMPONENT_NAME, DeploymentType, ) from clp_py_utils.clp_metadata_db_utils import ( get_archives_table_name, @@ -470,10 +470,15 @@ def deploy(self): check_docker_dependencies(should_compose_run=False) self._provision() - logger.info(f"Starting CLP using Docker Compose...") + deployment_type = self.clp_config.get_deployment_type() + logger.info(f"Starting CLP using Docker Compose ({deployment_type})...") + cmd = ["docker", "compose"] + if deployment_type == DeploymentType.BASE: + cmd += ["--file", "docker-compose.base.yaml"] + cmd += ["up", "--detach"] try: subprocess.run( - ["docker", "compose", "up", "-d"], + cmd, cwd=self.clp_home, stderr=subprocess.STDOUT, check=True, diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index e431df85bc..691623321c 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -28,45 +28,6 @@ WEBUI_COMPONENT_NAME = "webui" GARBAGE_COLLECTOR_COMPONENT_NAME = "garbage_collector" -# Component groups -GENERAL_SCHEDULING_COMPONENTS = { - QUEUE_COMPONENT_NAME, - REDIS_COMPONENT_NAME, -} -COMPRESSION_COMPONENTS = GENERAL_SCHEDULING_COMPONENTS | { - DB_COMPONENT_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, -} -QUERY_COMPONENTS = GENERAL_SCHEDULING_COMPONENTS | { - DB_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - REDUCER_COMPONENT_NAME, -} -UI_COMPONENTS = { - RESULTS_CACHE_COMPONENT_NAME, - WEBUI_COMPONENT_NAME, -} -STORAGE_MANAGEMENT_COMPONENTS = {GARBAGE_COLLECTOR_COMPONENT_NAME} -ALL_COMPONENTS = ( - COMPRESSION_COMPONENTS | QUERY_COMPONENTS | UI_COMPONENTS | STORAGE_MANAGEMENT_COMPONENTS -) - -# Target names -ALL_TARGET_NAME = "" -CONTROLLER_TARGET_NAME = "controller" - -TARGET_TO_COMPONENTS = { - ALL_TARGET_NAME: ALL_COMPONENTS, - CONTROLLER_TARGET_NAME: GENERAL_SCHEDULING_COMPONENTS - | { - COMPRESSION_SCHEDULER_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - } - | STORAGE_MANAGEMENT_COMPONENTS, -} - # Action names ARCHIVE_MANAGER_ACTION_NAME = "archive_manager" @@ -101,6 +62,11 @@ CLP_REDIS_PASS_ENV_VAR_NAME = "CLP_REDIS_PASS" +class DeploymentType(KebabCaseStrEnum): + BASE = auto() + FULL = auto() + + class StorageEngine(KebabCaseStrEnum): CLP = auto() CLP_S = auto() @@ -1001,6 +967,12 @@ def load_execution_container_name(self): def get_shared_config_file_path(self) -> pathlib.Path: return self.logs_directory / CLP_SHARED_CONFIG_FILENAME + def get_deployment_type(self) -> DeploymentType: + if QueryEngine.PRESTO == self.package.query_engine: + return DeploymentType.BASE + else: + return DeploymentType.FULL + def dump_to_primitive_dict(self): custom_serialized_fields = { "database", diff --git a/taskfile.yaml b/taskfile.yaml index 231c83849b..651e001c6e 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -116,7 +116,7 @@ tasks: - "components/clp-py-utils/dist/*.whl" - "components/job-orchestration/dist/*.whl" - "components/package-template/src/**/*" - - "tools/deployment/package/docker-compose.yaml" + - "tools/deployment/package/**/*" generates: ["{{.CHECKSUM_FILE}}"] deps: - "core" @@ -166,7 +166,7 @@ tasks: PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm ci --omit=dev - >- rsync -a - "tools/deployment/package/docker-compose.yaml" + "tools/deployment/package/" "{{.OUTPUT_DIR}}" # This command must be last - task: "utils:checksum:compute" diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml new file mode 100644 index 0000000000..a49e4d3cd5 --- /dev/null +++ b/tools/deployment/package/docker-compose.base.yaml @@ -0,0 +1,293 @@ +name: "clp-package-base" + +# Common service defaults. +x-service-defaults: &service_defaults + networks: ["clp-network"] + stop_grace_period: "3s" + user: "${CLP_UID_GID:-1000:1000}" + +# Common healthcheck defaults. +x-healthcheck-defaults: &healthcheck_defaults + interval: "30s" + retries: 3 + start_interval: "1s" + start_period: "10s" + timeout: "10s" + +networks: + clp-network: + driver: "bridge" + +secrets: + CLP_DB_PASS_FILE: + environment: "CLP_DB_PASS" + +services: + db: + <<: *service_defaults + container_name: "database" + image: "${CLP_DB_IMAGE:-mysql:8.0.23}" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + environment: + MYSQL_DATABASE: "${CLP_DB_NAME}" + MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_USER: "${CLP_DB_USER}" + secrets: + - "CLP_DB_PASS_FILE" + ports: + - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" + volumes: + - "${CLP_DB_CONF_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" + - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" + - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "mysqladmin", "ping", + "--silent", + "-h", "127.0.0.1", + "-u", "${CLP_DB_USER}", + "--password=${CLP_DB_PASS}" + ] + + db-table-creator: + <<: *service_defaults + container_name: "db_table_creator" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + volumes: + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" + depends_on: + db: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "clp_py_utils.create-db-tables", + "--config", "/etc/clp-config.yml", + "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE}" + ] + + queue: + <<: *service_defaults + container_name: "queue" + image: "rabbitmq:3.9.8" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + environment: + RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" + RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" + RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" + ports: + - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" + volumes: + - "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}:/var/log/rabbitmq" + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "rabbitmq-diagnostics", "check_running" + ] + + redis: + <<: *service_defaults + container_name: "redis" + image: "redis:7.2.4" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + ports: + - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" + volumes: + - "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}:/usr/local/etc/redis/redis.conf:ro" + - "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}:/data" + - "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}:/var/log/redis" + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "redis-cli", + "-h", "127.0.0.1", + "-p", "6379", + "-a", "${CLP_REDIS_PASS}", + "PING" + ] + command: [ + "redis-server", + "/usr/local/etc/redis/redis.conf", + "--requirepass", "${CLP_REDIS_PASS}" + ] + + results-cache: + <<: *service_defaults + container_name: "results_cache" + image: "mongo:7.0.1" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + ports: + - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" + volumes: + - "${CLP_RESULTS_CACHE_CONF_DIR_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" + - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" + - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" + healthcheck: + <<: *healthcheck_defaults + test: >- + echo 'db.runCommand("ping").ok' | + mongosh 127.0.0.1:27017/test --quiet + command: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0", + ] + + results-cache-indices-creator: + <<: *service_defaults + container_name: "results_cache_indices_creator" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + depends_on: + results-cache: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "clp_py_utils.initialize-results-cache", + "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", + "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", + ] + + compression-scheduler: + <<: *service_defaults + container_name: "compression_scheduler" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + volumes: + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}\ +:/var/log/compression_scheduler.log" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "/:/mnt/logs:ro" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + queue: + condition: "service_healthy" + redis: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "job_orchestration.scheduler.compress.compression_scheduler", + "--config", "/etc/clp-config.yml" + ] + + compression-worker: + <<: *service_defaults + container_name: "compression_worker" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + CLP_CONFIG_PATH: "/etc/clp-config.yml" + CLP_HOME: "/opt/clp" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/compression_worker" + CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + volumes: + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ +/var/log/compression_worker" + - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "/:/mnt/logs:ro" + command: [ + "python3", + "-u", + "/opt/clp/lib/python3/site-packages/bin/celery", + "-A", "job_orchestration.executor.compress", + "worker", + "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", + "--loglevel", "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}", + "-f", "/var/log/compression_worker/worker.log", + "-Q", "compression", + "-n", "compression-worker" + ] + + webui: + <<: *service_defaults + container_name: "webui" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + HOST: "0.0.0.0" + NODE_ENV: "production" + NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" + PORT: "4000" + RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" + ports: + - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" + volumes: + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" + - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" + - "./var/www/webui/server/dist/settings.json\ +:/opt/clp/var/www/webui/server/dist/settings.json:ro" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + results-cache-indices-creator: + condition: "service_completed_successfully" + command: [ + "/opt/clp/bin/node-22", + "/opt/clp/var/www/webui/server/dist/src/main.js" + ] + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "bash", + "-c", + "< /dev/tcp/webui/4000" + ] + + garbage-collector: + <<: *service_defaults + container_name: "garbage_collector" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_HOME: "/opt/clp" + CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/garbage_collector" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + volumes: + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + results-cache-indices-creator: + condition: "service_completed_successfully" + command: [ + "python3", "-u", + "-m", "job_orchestration.garbage_collector.garbage_collector", + "--config", "/etc/clp-config.yml", + ] diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 64aa7232eb..5e60c09797 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -1,3 +1,8 @@ +name: "clp-package" + +include: + - "docker-compose.base.yaml" + # Common service defaults. x-service-defaults: &service_defaults networks: ["clp-network"] @@ -12,185 +17,7 @@ x-healthcheck-defaults: &healthcheck_defaults start_period: "10s" timeout: "10s" -networks: - clp-network: - driver: "bridge" - -name: "clp-package" - -secrets: - CLP_DB_PASS_FILE: - environment: "CLP_DB_PASS" - services: - db: - <<: *service_defaults - container_name: "database" - image: "${CLP_DB_IMAGE:-mysql:8.0.23}" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" - environment: - MYSQL_DATABASE: "${CLP_DB_NAME}" - MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" - MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" - MYSQL_USER: "${CLP_DB_USER}" - secrets: - - "CLP_DB_PASS_FILE" - ports: - - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" - volumes: - - "${CLP_DB_CONF_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" - - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" - - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" - healthcheck: - <<: *healthcheck_defaults - test: [ - "CMD", - "mysqladmin", "ping", - "--silent", - "-h", "127.0.0.1", - "-u", "${CLP_DB_USER}", - "--password=${CLP_DB_PASS}" - ] - - db-table-creator: - <<: *service_defaults - container_name: "db_table_creator" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" - depends_on: - db: - condition: "service_healthy" - command: [ - "python3", - "-u", - "-m", "clp_py_utils.create-db-tables", - "--config", "/etc/clp-config.yml", - "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE}" - ] - - queue: - <<: *service_defaults - container_name: "queue" - image: "rabbitmq:3.9.8" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" - environment: - RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" - RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" - RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" - ports: - - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" - volumes: - - "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}:/var/log/rabbitmq" - healthcheck: - <<: *healthcheck_defaults - test: [ - "CMD", - "rabbitmq-diagnostics", "check_running" - ] - - redis: - <<: *service_defaults - container_name: "redis" - image: "redis:7.2.4" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" - ports: - - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" - volumes: - - "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}:/usr/local/etc/redis/redis.conf:ro" - - "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}:/data" - - "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}:/var/log/redis" - healthcheck: - <<: *healthcheck_defaults - test: [ - "CMD", - "redis-cli", - "-h", "127.0.0.1", - "-p", "6379", - "-a", "${CLP_REDIS_PASS}", - "PING" - ] - command: [ - "redis-server", - "/usr/local/etc/redis/redis.conf", - "--requirepass", "${CLP_REDIS_PASS}" - ] - - results-cache: - <<: *service_defaults - container_name: "results_cache" - image: "mongo:7.0.1" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" - ports: - - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" - volumes: - - "${CLP_RESULTS_CACHE_CONF_DIR_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" - - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" - - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" - healthcheck: - <<: *healthcheck_defaults - test: >- - echo 'db.runCommand("ping").ok' | - mongosh 127.0.0.1:27017/test --quiet - command: [ - "--config", "/etc/mongo/mongod.conf", - "--bind_ip", "0.0.0.0", - ] - - results-cache-indices-creator: - <<: *service_defaults - container_name: "results_cache_indices_creator" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - depends_on: - results-cache: - condition: "service_healthy" - command: [ - "python3", - "-u", - "-m", "clp_py_utils.initialize-results-cache", - "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", - "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", - ] - - compression-scheduler: - <<: *service_defaults - container_name: "compression_scheduler" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" - CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" - CLP_LOGS_DIR: "/var/log" - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} - volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}\ -:/var/log/compression_scheduler.log" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "/:/mnt/logs:ro" - depends_on: - db-table-creator: - condition: "service_completed_successfully" - queue: - condition: "service_healthy" - redis: - condition: "service_healthy" - command: [ - "python3", - "-u", - "-m", "job_orchestration.scheduler.compress.compression_scheduler", - "--config", "/etc/clp-config.yml" - ] - query-scheduler: <<: *service_defaults container_name: "query_scheduler" @@ -231,44 +58,6 @@ services: "< /dev/tcp/query_scheduler/7000" ] - compression-worker: - <<: *service_defaults - container_name: "compression_worker" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - CLP_CONFIG_PATH: "/etc/clp-config.yml" - CLP_HOME: "/opt/clp" - CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" - CLP_LOGS_DIR: "/var/log/compression_worker" - CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} - volumes: - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ -/var/log/compression_worker" - - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "/:/mnt/logs:ro" - command: [ - "python3", - "-u", - "/opt/clp/lib/python3/site-packages/bin/celery", - "-A", "job_orchestration.executor.compress", - "worker", - "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", - "--loglevel", "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}", - "-f", "/var/log/compression_worker/worker.log", - "-Q", "compression", - "-n", "compression-worker" - ] - query-worker: <<: *service_defaults container_name: "query_worker" @@ -329,66 +118,3 @@ services: "--concurrency", "${CLP_REDUCER_CONCURRENCY:-1}", "--upsert-interval", "${CLP_REDUCER_UPSERT_INTERVAL:-100}" ] - - webui: - <<: *service_defaults - container_name: "webui" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" - HOST: "0.0.0.0" - NODE_ENV: "production" - NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" - PORT: "4000" - RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" - ports: - - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" - volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" - - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - - "./var/www/webui/server/dist/settings.json\ -:/opt/clp/var/www/webui/server/dist/settings.json:ro" - depends_on: - db-table-creator: - condition: "service_completed_successfully" - results-cache-indices-creator: - condition: "service_completed_successfully" - command: [ - "/opt/clp/bin/node-22", - "/opt/clp/var/www/webui/server/dist/src/main.js" - ] - healthcheck: - <<: *healthcheck_defaults - test: [ - "CMD", - "bash", - "-c", - "< /dev/tcp/webui/4000" - ] - - garbage-collector: - <<: *service_defaults - container_name: "garbage_collector" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" - CLP_HOME: "/opt/clp" - CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" - CLP_LOGS_DIR: "/var/log/garbage_collector" - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" - depends_on: - db-table-creator: - condition: "service_completed_successfully" - results-cache-indices-creator: - condition: "service_completed_successfully" - command: [ - "python3", "-u", - "-m", "job_orchestration.garbage_collector.garbage_collector", - "--config", "/etc/clp-config.yml", - ] From e506f6db72a84cd22dbc7b00a94a024b56875252 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 24 Sep 2025 19:31:03 -0400 Subject: [PATCH 088/238] feat(controller): Add instance ID support for Docker Compose project names. --- .../clp_package_utils/controller.py | 36 +++++++++++++++---- .../clp_package_utils/general.py | 5 +-- .../clp_package_utils/scripts/start_clp.py | 7 ++-- .../clp_package_utils/scripts/stop_clp.py | 5 +-- .../package/docker-compose.base.yaml | 1 + 5 files changed, 40 insertions(+), 14 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index ef9a1870f8..407937766b 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -6,6 +6,7 @@ import socket import stat import subprocess +import uuid from abc import ABC, abstractmethod from typing import Any, Dict @@ -18,6 +19,7 @@ COMPRESSION_SCHEDULER_COMPONENT_NAME, COMPRESSION_WORKER_COMPONENT_NAME, DB_COMPONENT_NAME, + DeploymentType, GARBAGE_COLLECTOR_COMPONENT_NAME, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, @@ -28,7 +30,7 @@ RESULTS_CACHE_COMPONENT_NAME, StorageEngine, StorageType, - WEBUI_COMPONENT_NAME, DeploymentType, + WEBUI_COMPONENT_NAME, ) from clp_py_utils.clp_metadata_db_utils import ( get_archives_table_name, @@ -457,7 +459,8 @@ class DockerComposeController(BaseController): Controller for deploying CLP components using Docker Compose. """ - def __init__(self, clp_config: CLPConfig): + def __init__(self, clp_config: CLPConfig, instance_id: str): + self._project_name = f"clp-package-{instance_id}" super().__init__(clp_config) def deploy(self): @@ -467,12 +470,13 @@ def deploy(self): 2. Provisioning environment variables and configuration. 3. Running `docker compose up -d`. """ - check_docker_dependencies(should_compose_run=False) + check_docker_dependencies(should_compose_run=False, project_name=self._project_name) self._provision() deployment_type = self.clp_config.get_deployment_type() logger.info(f"Starting CLP using Docker Compose ({deployment_type})...") - cmd = ["docker", "compose"] + + cmd = ["docker", "compose", "--project-name", self._project_name] if deployment_type == DeploymentType.BASE: cmd += ["--file", "docker-compose.base.yaml"] cmd += ["up", "--detach"] @@ -491,12 +495,12 @@ def stop(self): """ Stops CLP components deployed via Docker Compose. """ - check_docker_dependencies(should_compose_run=True) + check_docker_dependencies(should_compose_run=True, project_name=self._project_name) logger.info("Stopping all CLP containers using Docker Compose...") try: subprocess.run( - ["docker", "compose", "down"], + ["docker", "compose", "--project-name", self._project_name, "down"], cwd=self.clp_home, stderr=subprocess.STDOUT, check=True, @@ -566,6 +570,26 @@ def _provision(self): env_file.write(f"{key}={value}\n") +def get_or_create_instance_id(clp_config: CLPConfig): + """ + Gets or create a unique instance ID for this CLP instance. + :param clp_config: + :return: The instance ID. + """ + instance_id_file_path = clp_config.logs_directory / "instance-id" + + if instance_id_file_path.exists(): + with open(instance_id_file_path, "r") as f: + instance_id = f.readline() + else: + instance_id = str(uuid.uuid4())[-4:] + with open(instance_id_file_path, "w") as f: + f.write(instance_id) + f.flush() + + return instance_id + + def _chown_paths_if_root(*paths: pathlib.Path): """ Changes ownership of the given paths to the default service container user/group IDs if the diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index fd357aad36..b7dcf14936 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -148,11 +148,12 @@ def is_docker_compose_running(project_name: str) -> bool: raise EnvironmentError("docker-compose is not installed or not functioning properly.") -def check_docker_dependencies(should_compose_run: bool = False): +def check_docker_dependencies(should_compose_run: bool, project_name: str): """ Checks if Docker and Docker Compose are installed, and whether Docker Compose is running or not. :param should_compose_run: + :param project_name: The Docker Compose project name to check. :raises EnvironmentError: If any Docker dependency is not installed or Docker Compose state does not match expectation. """ @@ -167,7 +168,7 @@ def check_docker_dependencies(should_compose_run: bool = False): except subprocess.CalledProcessError: raise EnvironmentError("docker is not installed or available on the path") - is_running = is_docker_compose_running("clp-package") + is_running = is_docker_compose_running(project_name) if should_compose_run and not is_running: raise EnvironmentError("docker-compose is not running.") if not should_compose_run and is_running: diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index b4e4a3449c..78fc6739a8 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -5,10 +5,8 @@ from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH -from clp_package_utils.controller import DockerComposeController +from clp_package_utils.controller import DockerComposeController, get_or_create_instance_id from clp_package_utils.general import ( - dump_shared_container_config, - generate_docker_compose_container_config, get_clp_home, load_config_file, validate_and_load_db_credentials_file, @@ -61,8 +59,9 @@ def main(argv): logger.exception("Failed to load config.") return -1 + instance_id = get_or_create_instance_id(clp_config) try: - controller = DockerComposeController(clp_config) + controller = DockerComposeController(clp_config, instance_id) controller.deploy() except Exception as ex: if type(ex) == ValueError: diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index f0d54e6de5..5506e81f08 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -3,7 +3,7 @@ from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH -from clp_package_utils.controller import DockerComposeController +from clp_package_utils.controller import DockerComposeController, get_or_create_instance_id from clp_package_utils.general import ( get_clp_home, load_config_file, @@ -22,8 +22,9 @@ def main(): logger.exception("Failed to load config.") return -1 + instance_id = get_or_create_instance_id(clp_config) try: - controller = DockerComposeController(clp_config) + controller = DockerComposeController(clp_config, instance_id) controller.stop() except: logger.exception("Failed to stop CLP.") diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index a49e4d3cd5..e259afb985 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -17,6 +17,7 @@ x-healthcheck-defaults: &healthcheck_defaults networks: clp-network: driver: "bridge" + name: "clp-network" secrets: CLP_DB_PASS_FILE: From 93034fe2a0a43335e0eaef9d21cd62be296f8d84 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 24 Sep 2025 19:34:28 -0400 Subject: [PATCH 089/238] remove network to use default network with bridge driver --- tools/deployment/package/docker-compose.base.yaml | 6 ------ tools/deployment/package/docker-compose.yaml | 1 - 2 files changed, 7 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index e259afb985..01afde0fb9 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -2,7 +2,6 @@ name: "clp-package-base" # Common service defaults. x-service-defaults: &service_defaults - networks: ["clp-network"] stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" @@ -14,11 +13,6 @@ x-healthcheck-defaults: &healthcheck_defaults start_period: "10s" timeout: "10s" -networks: - clp-network: - driver: "bridge" - name: "clp-network" - secrets: CLP_DB_PASS_FILE: environment: "CLP_DB_PASS" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 5e60c09797..9e89a28a0d 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -5,7 +5,6 @@ include: # Common service defaults. x-service-defaults: &service_defaults - networks: ["clp-network"] stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" From ca620094ce7622be4711895f9827a13d32d2b886 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 24 Sep 2025 23:43:02 -0400 Subject: [PATCH 090/238] fix(controller): update clp_home to be private variable. --- .../clp_package_utils/controller.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 407937766b..1ee3c17aff 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -69,8 +69,8 @@ class BaseController(ABC): def __init__(self, clp_config: CLPConfig): self.clp_config = clp_config - self.clp_home = get_clp_home() - self._conf_dir = self.clp_home / "etc" + self._clp_home = get_clp_home() + self._conf_dir = self._clp_home / "etc" @abstractmethod def deploy(self): @@ -309,10 +309,10 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" client_settings_json_path = ( - self.clp_home / "var" / "www" / "webui" / "client" / "settings.json" + self._clp_home / "var" / "www" / "webui" / "client" / "settings.json" ) server_settings_json_path = ( - self.clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" + self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" ) validate_webui_config(self.clp_config, client_settings_json_path, server_settings_json_path) @@ -483,7 +483,7 @@ def deploy(self): try: subprocess.run( cmd, - cwd=self.clp_home, + cwd=self._clp_home, stderr=subprocess.STDOUT, check=True, ) @@ -501,7 +501,7 @@ def stop(self): try: subprocess.run( ["docker", "compose", "--project-name", self._project_name, "down"], - cwd=self.clp_home, + cwd=self._clp_home, stderr=subprocess.STDOUT, check=True, ) @@ -565,7 +565,7 @@ def _provision(self): if self.clp_config.aws_config_directory is not None: env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self.clp_config.aws_config_directory) - with open(f"{self.clp_home}/.env", "w") as env_file: + with open(f"{self._clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): env_file.write(f"{key}={value}\n") From 82cc48c3527306734e620b37cbdb0a0ab84d437f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 00:41:52 -0400 Subject: [PATCH 091/238] Use unique ids to launch dev clp package. --- .../clp-py-utils/clp_py_utils/clp_config.py | 22 ++++++++++++++----- taskfile.yaml | 1 + taskfiles/docker-images.yaml | 1 + tools/docker-images/clp-package/build.sh | 17 +++++++++++++- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 691623321c..33658baad5 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,6 +1,7 @@ import os import pathlib from enum import auto +from importlib.metadata import version from typing import ClassVar, Literal, Optional, Set, Union from pydantic import BaseModel, PrivateAttr, root_validator, validator @@ -36,8 +37,6 @@ COMPRESSION_JOBS_TABLE_NAME = "compression_jobs" COMPRESSION_TASKS_TABLE_NAME = "compression_tasks" -OS_RELEASE_FILE_PATH = pathlib.Path("etc") / "os-release" - # Paths CONTAINER_AWS_CONFIG_DIRECTORY = pathlib.Path("/") / ".aws" CONTAINER_CLP_HOME = pathlib.Path("/") / "opt" / "clp" @@ -52,7 +51,9 @@ CLP_DEFAULT_LOG_DIRECTORY_PATH = pathlib.Path("var") / "log" CLP_DEFAULT_DATASET_NAME = "default" CLP_METADATA_TABLE_PREFIX = "clp_" +CLP_PACKAGE_IMAGE_ID_PATH = pathlib.Path("image.id") CLP_SHARED_CONFIG_FILENAME = ".clp-config.yml" +CLP_VERSION_FILE_PATH = pathlib.Path("VERSION") # Environment variable names CLP_DB_USER_ENV_VAR_NAME = "CLP_DB_USER" @@ -861,7 +862,8 @@ class CLPConfig(BaseModel): logs_directory: pathlib.Path = CLP_DEFAULT_LOG_DIRECTORY_PATH aws_config_directory: Optional[pathlib.Path] = None - _os_release_file_path: pathlib.Path = PrivateAttr(default=OS_RELEASE_FILE_PATH) + _image_id_path: pathlib.Path = PrivateAttr(default=CLP_PACKAGE_IMAGE_ID_PATH) + _version_file_path: pathlib.Path = PrivateAttr(default=CLP_VERSION_FILE_PATH) def make_config_paths_absolute(self, clp_home: pathlib.Path): if StorageType.FS == self.logs_input.type: @@ -871,7 +873,8 @@ def make_config_paths_absolute(self, clp_home: pathlib.Path): self.stream_output.storage.make_config_paths_absolute(clp_home) self.data_directory = make_config_path_absolute(clp_home, self.data_directory) self.logs_directory = make_config_path_absolute(clp_home, self.logs_directory) - self._os_release_file_path = make_config_path_absolute(clp_home, self._os_release_file_path) + self._image_id_path = make_config_path_absolute(clp_home, self._image_id_path) + self._version_file_path = make_config_path_absolute(clp_home, self._version_file_path) def validate_logs_input_config(self): logs_input_type = self.logs_input.type @@ -959,10 +962,17 @@ def validate_aws_config_dir(self): def load_execution_container_name(self): if self.execution_container is not None: - # Accept configured value for releases + # Accept configured value for debug purposes return - self.execution_container = "clp-package:dev" + if self._image_id_path.exists(): + with open(self._image_id_path) as image_id_file: + self.execution_container = image_id_file.read().strip() + + if not bool(self.execution_container): + with open(self._version_file_path) as version_file: + package_version = version_file.read().strip() + self.execution_container = f"ghcr.io/y-scope/clp/clp-package:{package_version}" def get_shared_config_file_path(self) -> pathlib.Path: return self.logs_directory / CLP_SHARED_CONFIG_FILENAME diff --git a/taskfile.yaml b/taskfile.yaml index 347d89866b..f9933a5121 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -172,6 +172,7 @@ tasks: rsync -a "tools/deployment/package/" "{{.OUTPUT_DIR}}" + - "echo '{{.G_PACKAGE_VERSION}}' > '{{.OUTPUT_DIR}}/VERSION'" # This command must be last - task: "utils:checksum:compute" vars: diff --git a/taskfiles/docker-images.yaml b/taskfiles/docker-images.yaml index 9a546f713c..c3e6ca57b5 100644 --- a/taskfiles/docker-images.yaml +++ b/taskfiles/docker-images.yaml @@ -12,3 +12,4 @@ tasks: - ":package" cmds: - "./build.sh" + - "rsync -a 'image.id' '{{.G_PACKAGE_BUILD_DIR}}'" diff --git a/tools/docker-images/clp-package/build.sh b/tools/docker-images/clp-package/build.sh index 39df1a1105..f44e40bc1e 100755 --- a/tools/docker-images/clp-package/build.sh +++ b/tools/docker-images/clp-package/build.sh @@ -4,11 +4,26 @@ set -eu set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +iid_file="${script_dir}/image.id" repo_root=${script_dir}/../../../ +# Remove the previous image after the build to allow layer reuse. +prev_image_id="" +if [[ -f "$iid_file" ]]; then + prev_image_id=$(cat "$iid_file") +fi +cleanup() { + if [[ -n "$prev_image_id" ]] && docker image inspect "$prev_image_id" >/dev/null 2>&1; then + echo "Removing previous image $prev_image_id" + docker image remove "$prev_image_id" + fi +} +trap cleanup EXIT + build_cmd=( docker build - --tag "clp-package:dev" + --iidfile "$iid_file" + --tag "clp-package:dev-${USER}-$(date +%s)" "$repo_root" --file "${script_dir}/Dockerfile" ) From 8f6ba1bdf03b2d75144c4f82bd8c06f6dc20e11d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 00:47:56 -0400 Subject: [PATCH 092/238] remove trailing space --- docs/src/dev-docs/design-docker-compose.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index d8fa7800ad..0f58aca712 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -4,7 +4,7 @@ This document explains the technical details of CLP's Docker Compose implementat ## Overview -The Docker Compose implementation depends on a new controller architecture with a `BaseController` +The Docker Compose implementation depends on a new controller architecture with a `BaseController` abstract class and a `DockerComposeController` implementation. ## Architecture From 2aff301de7a8f588f11894db6fce16f0496ab72a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 00:53:59 -0400 Subject: [PATCH 093/238] remove `execution_container` field from clp-config.yml template --- components/package-template/src/etc/clp-config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/package-template/src/etc/clp-config.yml b/components/package-template/src/etc/clp-config.yml index 3433a3e3a0..736f9a2ad5 100644 --- a/components/package-template/src/etc/clp-config.yml +++ b/components/package-template/src/etc/clp-config.yml @@ -1,6 +1,3 @@ -## Docker image to use for CLP package execution. -#execution_container: "clp-package:dev" -# ## Location (e.g., directory) containing any logs you wish to compress. Must be reachable by all ## workers. #logs_input: From 5c91bf391075808eea1f2ee1684c4622f222fd09 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 00:55:07 -0400 Subject: [PATCH 094/238] fix(docker-compose): update `include` array syntax to use [] format. --- tools/deployment/package/docker-compose.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 9e89a28a0d..5e63b59884 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -1,7 +1,6 @@ name: "clp-package" -include: - - "docker-compose.base.yaml" +include: ["docker-compose.base.yaml"] # Common service defaults. x-service-defaults: &service_defaults From 049e269d1fe62b50b039ffdce8063112db1aa163 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 01:07:37 -0400 Subject: [PATCH 095/238] update service name from `db` to `database`. --- docs/src/dev-docs/design-docker-compose.md | 6 +++--- tools/deployment/package/docker-compose.base.yaml | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index 0f58aca712..add169870b 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -52,7 +52,7 @@ The Docker Compose setup includes the following services: :::{mermaid} graph LR %% Services - db["db (MySQL)"] + database["database (MySQL)"] queue["queue (RabbitMQ)"] redis["redis (Redis)"] results_cache["results-cache (MongoDB)"] @@ -69,7 +69,7 @@ graph LR results_cache_indices_creator["results-cache-indices-creator"] %% Dependencies - db -->|healthy| db_table_creator + database -->|healthy| db_table_creator results_cache -->|healthy| results_cache_indices_creator db_table_creator -->|completed_successfully| compression_scheduler queue -->|healthy| compression_scheduler @@ -85,7 +85,7 @@ graph LR results_cache_indices_creator -->|completed_successfully| garbage_collector subgraph Databases - db + database queue redis results_cache diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 01afde0fb9..2b5bcfdb6b 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -18,7 +18,7 @@ secrets: environment: "CLP_DB_PASS" services: - db: + database: <<: *service_defaults container_name: "database" image: "${CLP_DB_IMAGE:-mysql:8.0.23}" @@ -58,7 +58,7 @@ services: volumes: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" depends_on: - db: + database: condition: "service_healthy" command: [ "python3", @@ -173,7 +173,7 @@ services: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" depends_on: - db-table-creator: + database: condition: "service_completed_successfully" queue: condition: "service_healthy" @@ -245,7 +245,7 @@ services: - "./var/www/webui/server/dist/settings.json\ :/opt/clp/var/www/webui/server/dist/settings.json:ro" depends_on: - db-table-creator: + database: condition: "service_completed_successfully" results-cache-indices-creator: condition: "service_completed_successfully" @@ -277,7 +277,7 @@ services: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" depends_on: - db-table-creator: + database: condition: "service_completed_successfully" results-cache-indices-creator: condition: "service_completed_successfully" From fc70c054a1865d38db8e226d30d62741a3dbea40 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 01:09:44 -0400 Subject: [PATCH 096/238] fix(docker-compose, controller): correct configuration file variable name. --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- tools/deployment/package/docker-compose.base.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 1ee3c17aff..1b4132ad74 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -196,7 +196,7 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: _chown_paths_if_root(data_dir, logs_dir) return { - "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), + "CLP_RESULTS_CACHE_CONF_FILE_HOST": str(conf_file), "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self.clp_config.results_cache.host), diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 2b5bcfdb6b..628af466ae 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -123,7 +123,7 @@ services: ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" volumes: - - "${CLP_RESULTS_CACHE_CONF_DIR_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" + - "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" healthcheck: From 2cb1807234020cf6b734b9eaf14636de39dedf23 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 01:14:21 -0400 Subject: [PATCH 097/238] add _HOST postfix to path env var names for consistency --- components/clp-package-utils/clp_package_utils/controller.py | 4 ++-- tools/deployment/package/docker-compose.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 1b4132ad74..7e3d7067fa 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -273,7 +273,7 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: return { "CLP_QUERY_WORKER_LOGGING_LEVEL": self.clp_config.query_worker.logging_level, - "CLP_QUERY_WORKER_LOGS_DIR": str(logs_dir), + "CLP_QUERY_WORKER_LOGS_DIR_HOST": str(logs_dir), "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } @@ -292,7 +292,7 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: return { "CLP_REDUCER_LOGGING_LEVEL": self.clp_config.reducer.logging_level, - "CLP_REDUCER_LOGS_DIR": str(logs_dir), + "CLP_REDUCER_LOGS_DIR_HOST": str(logs_dir), "CLP_REDUCER_CONCURRENCY": str(num_workers), "CLP_REDUCER_UPSERT_INTERVAL": str(self.clp_config.reducer.upsert_interval), } diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 5e63b59884..319e11c694 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -76,7 +76,7 @@ services: - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_QUERY_WORKER_LOGS_DIR:-./var/log/query_worker}:/var/log/query_worker" + - "${CLP_QUERY_WORKER_LOGS_DIR_HOST:-./var/log/query_worker}:/var/log/query_worker" - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" command: [ @@ -103,7 +103,7 @@ services: PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_REDUCER_LOGS_DIR:-./var/log/reducer}:/var/log/reducer" + - "${CLP_REDUCER_LOGS_DIR_HOST:-./var/log/reducer}:/var/log/reducer" depends_on: query-scheduler: condition: "service_healthy" From e657feb111a86ac730a3b585536cacfc4078f030 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 23:31:53 -0400 Subject: [PATCH 098/238] fix: the webui service should depend on db-table-creator rather than database --- tools/deployment/package/docker-compose.base.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 628af466ae..a0d5e301ae 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -173,7 +173,7 @@ services: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" depends_on: - database: + db-table-creator: condition: "service_completed_successfully" queue: condition: "service_healthy" @@ -245,7 +245,7 @@ services: - "./var/www/webui/server/dist/settings.json\ :/opt/clp/var/www/webui/server/dist/settings.json:ro" depends_on: - database: + db-table-creator: condition: "service_completed_successfully" results-cache-indices-creator: condition: "service_completed_successfully" @@ -277,7 +277,7 @@ services: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" depends_on: - database: + db-table-creator: condition: "service_completed_successfully" results-cache-indices-creator: condition: "service_completed_successfully" From af96fc8606e8817319c1b8a9975a2eedd748120f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 1 Oct 2025 18:32:03 -0400 Subject: [PATCH 099/238] lint --- components/clp-package-utils/clp_package_utils/controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 7e3d7067fa..73b3986614 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -309,10 +309,10 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" client_settings_json_path = ( - self._clp_home / "var" / "www" / "webui" / "client" / "settings.json" + self._clp_home / "var" / "www" / "webui" / "client" / "settings.json" ) server_settings_json_path = ( - self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" + self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" ) validate_webui_config(self.clp_config, client_settings_json_path, server_settings_json_path) From dd56d04324ea7bad3135da6443e7ab7112939244 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 2 Oct 2025 18:01:52 -0400 Subject: [PATCH 100/238] fix: Replace container_name with hostname for services to avoid name conflicts with multiple of the projects starting on the same host. --- .../package/docker-compose.base.yaml | 20 +++++++++---------- tools/deployment/package/docker-compose.yaml | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index a0d5e301ae..62996dec0a 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -20,8 +20,8 @@ secrets: services: database: <<: *service_defaults - container_name: "database" image: "${CLP_DB_IMAGE:-mysql:8.0.23}" + hostname: "database" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" environment: MYSQL_DATABASE: "${CLP_DB_NAME}" @@ -49,8 +49,8 @@ services: db-table-creator: <<: *service_defaults - container_name: "db_table_creator" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "db_table_creator" environment: CLP_DB_PASS: "${CLP_DB_PASS}" CLP_DB_USER: "${CLP_DB_USER}" @@ -70,8 +70,8 @@ services: queue: <<: *service_defaults - container_name: "queue" image: "rabbitmq:3.9.8" + hostname: "queue" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" environment: RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" @@ -90,8 +90,8 @@ services: redis: <<: *service_defaults - container_name: "redis" image: "redis:7.2.4" + hostname: "redis" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" @@ -117,8 +117,8 @@ services: results-cache: <<: *service_defaults - container_name: "results_cache" image: "mongo:7.0.1" + hostname: "results_cache" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" @@ -138,8 +138,8 @@ services: results-cache-indices-creator: <<: *service_defaults - container_name: "results_cache_indices_creator" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "results_cache_indices_creator" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" depends_on: @@ -155,8 +155,8 @@ services: compression-scheduler: <<: *service_defaults - container_name: "compression_scheduler" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "compression_scheduler" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS}" @@ -188,8 +188,8 @@ services: compression-worker: <<: *service_defaults - container_name: "compression_worker" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "compression_worker" environment: AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" @@ -226,8 +226,8 @@ services: webui: <<: *service_defaults - container_name: "webui" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "webui" environment: CLP_DB_PASS: "${CLP_DB_PASS}" CLP_DB_USER: "${CLP_DB_USER}" @@ -264,8 +264,8 @@ services: garbage-collector: <<: *service_defaults - container_name: "garbage_collector" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "garbage_collector" environment: CLP_DB_PASS: "${CLP_DB_PASS}" CLP_DB_USER: "${CLP_DB_USER}" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 319e11c694..7d938a238b 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -18,8 +18,8 @@ x-healthcheck-defaults: &healthcheck_defaults services: query-scheduler: <<: *service_defaults - container_name: "query_scheduler" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "query_scheduler" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS}" @@ -58,8 +58,8 @@ services: query-worker: <<: *service_defaults - container_name: "query_worker" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "query_worker" environment: AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" @@ -94,8 +94,8 @@ services: reducer: <<: *service_defaults - container_name: "reducer" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "reducer" environment: CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_REDUCER_LOGGING_LEVEL:-INFO}" From 5298540ba24630873fff6dd7ed36b0e9635352c2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 2 Oct 2025 18:09:54 -0400 Subject: [PATCH 101/238] refactor(docker-compose): move image definition to service defaults. --- tools/deployment/package/docker-compose.base.yaml | 7 +------ tools/deployment/package/docker-compose.yaml | 3 +-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 62996dec0a..a78168376e 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -2,6 +2,7 @@ name: "clp-package-base" # Common service defaults. x-service-defaults: &service_defaults + image: "${CLP_PACKAGE_CONTAINER}" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" @@ -49,7 +50,6 @@ services: db-table-creator: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "db_table_creator" environment: CLP_DB_PASS: "${CLP_DB_PASS}" @@ -138,7 +138,6 @@ services: results-cache-indices-creator: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "results_cache_indices_creator" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -155,7 +154,6 @@ services: compression-scheduler: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "compression_scheduler" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" @@ -188,7 +186,6 @@ services: compression-worker: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "compression_worker" environment: AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" @@ -226,7 +223,6 @@ services: webui: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "webui" environment: CLP_DB_PASS: "${CLP_DB_PASS}" @@ -264,7 +260,6 @@ services: garbage-collector: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "garbage_collector" environment: CLP_DB_PASS: "${CLP_DB_PASS}" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 7d938a238b..71c5492f3d 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -4,6 +4,7 @@ include: ["docker-compose.base.yaml"] # Common service defaults. x-service-defaults: &service_defaults + image: "${CLP_PACKAGE_CONTAINER}" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" @@ -18,7 +19,6 @@ x-healthcheck-defaults: &healthcheck_defaults services: query-scheduler: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "query_scheduler" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" @@ -58,7 +58,6 @@ services: query-worker: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "query_worker" environment: AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" From 140187f550d4d61653239ed9109e902b281f9d6e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 2 Oct 2025 18:14:36 -0400 Subject: [PATCH 102/238] refactor(docker-compose): Remove image entry from reducer service. --- tools/deployment/package/docker-compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 71c5492f3d..94aef25e42 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -93,7 +93,6 @@ services: reducer: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "reducer" environment: CLP_HOME: "/opt/clp" From 68586f014b8cc721f0ec8f513d26ed57a9e83450 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 2 Oct 2025 18:16:49 -0400 Subject: [PATCH 103/238] refactor(docker-compose): add default fallback for image and storage engine variables. --- tools/deployment/package/docker-compose.base.yaml | 4 ++-- tools/deployment/package/docker-compose.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index a78168376e..07b760e649 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -2,7 +2,7 @@ name: "clp-package-base" # Common service defaults. x-service-defaults: &service_defaults - image: "${CLP_PACKAGE_CONTAINER}" + image: "${CLP_PACKAGE_CONTAINER:-clp-package}" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" @@ -65,7 +65,7 @@ services: "-u", "-m", "clp_py_utils.create-db-tables", "--config", "/etc/clp-config.yml", - "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE}" + "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE:-clp}", ] queue: diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 94aef25e42..24ad6b242f 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -4,7 +4,7 @@ include: ["docker-compose.base.yaml"] # Common service defaults. x-service-defaults: &service_defaults - image: "${CLP_PACKAGE_CONTAINER}" + image: "${CLP_PACKAGE_CONTAINER:-clp-package}" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" From cba044a12f17420ac5f89afef1234fa22e656f67 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 2 Oct 2025 21:55:42 -0400 Subject: [PATCH 104/238] improve docs --- docs/src/dev-docs/design-docker-compose.md | 77 +++++++++++++++------- 1 file changed, 52 insertions(+), 25 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index add169870b..e365decd63 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -4,46 +4,71 @@ This document explains the technical details of CLP's Docker Compose implementat ## Overview -The Docker Compose implementation depends on a new controller architecture with a `BaseController` -abstract class and a `DockerComposeController` implementation. +The Docker Compose implementation follows a controller architecture with a `BaseController` abstract class and a +`DockerComposeController` implementation. ## Architecture -### Controller +### Controller Pattern -The orchestration implementation uses a controller pattern: +The orchestration uses a controller pattern: * `BaseController` (abstract): Defines the interface for provisioning and managing CLP components. -* `DockerComposeController`: Implements the Docker Compose-specific logic. +* `DockerComposeController`: Implements Docker Compose-specific logic. -### Initialization +## Initialization -1. **Provisioning Methods**: Each CLP component has a dedicated provisioning method in the - controller: `provision_()`. -2. **Environment Generation**: The controller generates a `.env` file with all necessary environment - variables for Docker Compose. -3. **Configuration Transformation**: The `transform_for_container_config()` method in `CLPConfig` - and related classes adapts the configuration for containerized environments. +The controller performs these initialization steps: -## Docker Compose File +1. **Provisioning**: Provisions all components and generates component specific configuration variables. +2. **Configuration Transformation**: The `transform_for_container()` method in `CLPConfig` adapts configurations for + containerized environments +3. **Environment Generation**: Creates a `.env` file with necessary Docker Compose variables -The `docker-compose.yaml` file defines all services with: +### Configuration Transformation -* Proper service dependencies using `depends_on` -* Health checks for critical services -* Volume mounts for persistent data -* Network configuration -* User permissions -* Resource limits +The `transform_for_container()` method in the `CLPConfig` class and related component classes adapts the configuration for containerized environments by: + +1. Converting host paths to container paths +2. Updating service hostnames to match Docker Compose service names +3. Setting appropriate ports for container communication + +### Environment Variables + +The controller generates a comprehensive set of environment variables that are written to a `.env` file, including: + +* Component-specific settings (ports, logging levels, concurrency) +* Credentials for database, queue, and Redis services +* Paths for data, logs, archives, and streams +* AWS credentials when needed ## Deployment Process -The `start-clp.py` script performs the following steps: +The `start-clp.sh` script executes the `start_clp.py` Python script to orchestrate the deployment. + +### Deployment Types + +CLP supports two deployment types determined by the `package.query_engine` configuration setting: -1. **Configuration Loading**: The start script loads and validates the CLP configuration. -2. **Provisioning**: The controller provisions all components and generates environment variables. -3. **Environment File Generation**: A `.env` file is created with all necessary variables. -4. **Docker Compose Execution**: `docker compose up -d` is executed to start all services. +1. **BASE**: For deployments using [Presto][presto-integration] as the query engine. Uses only + `docker-compose.base.yaml`. +2. **FULL**: For deployments using CLP's native query engine. Uses both compose files. + +## Docker Compose Files + +The Docker Compose setup uses two files: + +* `docker-compose.base.yaml`: Defines base services for all deployment types, excluding Celery scheduler and worker + components to allow separate Presto [integration][presto-integration]. +* `docker-compose.yaml`: Extends the base file with additional services for complete deployments + +Each file defines services with: + +* Service dependencies via `depends_on` +* Health checks for critical services +* Volume binding mounts for persistent data +* Network configuration +* User permissions ## Service architecture @@ -166,3 +191,5 @@ If you encounter issues with the Docker Compose deployment: ```bash docker compose config ``` + +[presto-integration]: ../user-docs/guides-using-presto.md \ No newline at end of file From f41e22abec53c3a384b377cd8b740ad787ef2b45 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 2 Oct 2025 22:09:02 -0400 Subject: [PATCH 105/238] refactor(docker-compose): use list syntax for healthcheck test command. --- tools/deployment/package/docker-compose.base.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 07b760e649..35ff1a6635 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -128,9 +128,10 @@ services: - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" healthcheck: <<: *healthcheck_defaults - test: >- - echo 'db.runCommand("ping").ok' | - mongosh 127.0.0.1:27017/test --quiet + test: [ + "CMD-SHELL", + "echo 'db.runCommand(\"ping\").ok' | mongosh 127.0.0.1:27017/test --quiet" + ] command: [ "--config", "/etc/mongo/mongod.conf", "--bind_ip", "0.0.0.0", From 02fcbe7b2f853c57d1e87fbe65a39d9202cf88ed Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 14:47:09 -0400 Subject: [PATCH 106/238] fix(docker-compose): revert workers log level to hardcoded "warning". --- tools/deployment/package/docker-compose.base.yaml | 2 +- tools/deployment/package/docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 35ff1a6635..68ec1e5c9d 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -216,7 +216,7 @@ services: "-A", "job_orchestration.executor.compress", "worker", "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", - "--loglevel", "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}", + "--loglevel", "WARNING", "-f", "/var/log/compression_worker/worker.log", "-Q", "compression", "-n", "compression-worker" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 24ad6b242f..e793d08dfe 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -85,7 +85,7 @@ services: "-A", "job_orchestration.executor.query", "worker", "--concurrency", "${CLP_QUERY_WORKER_CONCURRENCY:-1}", - "--loglevel", "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}", + "--loglevel", "WARNING", "-f", "/var/log/query_worker/worker.log", "-Q", "query", "-n", "query-worker" From 6b0e1f25cc5735c907992634be4fc4d2d6b79c64 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:20:07 -0400 Subject: [PATCH 107/238] fix(docker-compose): Fix environment variable name for results cache uri. --- tools/deployment/package/docker-compose.base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 68ec1e5c9d..11d00dbd9d 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -149,7 +149,7 @@ services: "python3", "-u", "-m", "clp_py_utils.initialize-results-cache", - "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", + "--uri", "mongodb://results_cache:27017/${CLP_RESULTS_CACHE_DB_NAME:-clp-query-results}", "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", ] From ddcf760d73cd25116929ab011bf86184be7c9596 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:35:22 -0400 Subject: [PATCH 108/238] refactor(clp-package-utils): Move EnvVarsDict type alias below imports. --- .../clp-package-utils/clp_package_utils/controller.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index b0348eac85..6923f095d2 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -10,9 +10,6 @@ from abc import ABC, abstractmethod from typing import Any, Dict -# Type alias for environment variables dictionary. -EnvVarsDict = Dict[str, str] - from clp_py_utils.clp_config import ( AwsAuthType, CLPConfig, @@ -51,6 +48,9 @@ validate_webui_config, ) +# Type alias for environment variables dictionary. +EnvVarsDict = Dict[str, str] + LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH DEFAULT_UID_GID = f"{os.getuid()}:{os.getgid()}" SERVICE_CONTAINER_USER_ID = 999 From 4134b1a43a4fbc0da762242a418adbd28e983f62 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:38:01 -0400 Subject: [PATCH 109/238] refactor(clp-package-utils): Rename LOGS_FILE_MODE to LOG_FILE_ACCESS_MODE for clarity. --- .../clp-package-utils/clp_package_utils/controller.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 6923f095d2..289a07710f 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -51,7 +51,8 @@ # Type alias for environment variables dictionary. EnvVarsDict = Dict[str, str] -LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH +LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + DEFAULT_UID_GID = f"{os.getuid()}:{os.getgid()}" SERVICE_CONTAINER_USER_ID = 999 SERVICE_CONTAINER_GROUP_ID = 999 @@ -215,7 +216,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_file = self.clp_config.logs_directory / f"{component_name}.log" - logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) + logs_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) return { "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self.clp_config.compression_scheduler.logging_level, @@ -232,7 +233,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_file = self.clp_config.logs_directory / f"{component_name}.log" - logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) + logs_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) return { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self.clp_config.query_scheduler.logging_level, From f483abda2ee5f59e9a47f083b128b5260efc4da7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:39:39 -0400 Subject: [PATCH 110/238] reflow comments to 100 char per line --- components/clp-package-utils/clp_package_utils/controller.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 289a07710f..e0efcf404d 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -63,9 +63,8 @@ class BaseController(ABC): """ - Abstract base controller for preparing and deploying CLP components. - Provides common logic for preparing environment variables, directories, - and configuration files for each service. + Abstract base controller for preparing and deploying CLP components. Provides common logic for + preparing environment variables, directories, and configuration files for each service. """ def __init__(self, clp_config: CLPConfig): From 7d4f47d813460bd1ac5d2fea53ebc090d1152815 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:40:17 -0400 Subject: [PATCH 111/238] refactor(clp-package-utils): Rename deploy method to start and adjust script usage. --- .../clp-package-utils/clp_package_utils/controller.py | 6 +++--- .../clp_package_utils/scripts/start_clp.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index e0efcf404d..a447bf814e 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -73,9 +73,9 @@ def __init__(self, clp_config: CLPConfig): self._conf_dir = self._clp_home / "etc" @abstractmethod - def deploy(self): + def start(self): """ - Deploys the provisioned components with orchestrator-specific logic. + Starts the provisioned components with orchestrator-specific logic. """ pass @@ -463,7 +463,7 @@ def __init__(self, clp_config: CLPConfig, instance_id: str): self._project_name = f"clp-package-{instance_id}" super().__init__(clp_config) - def deploy(self): + def start(self): """ Deploys CLP components using Docker Compose by: 1. Checking Docker dependencies. diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 78fc6739a8..368612530f 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -62,7 +62,7 @@ def main(argv): instance_id = get_or_create_instance_id(clp_config) try: controller = DockerComposeController(clp_config, instance_id) - controller.deploy() + controller.start() except Exception as ex: if type(ex) == ValueError: logger.error(f"Failed to start CLP: {ex}") From 064f36512da63585d68ba06d08402e69bd7fd98f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:41:24 -0400 Subject: [PATCH 112/238] refactor(clp-package-utils): Rename _provision method to _set_up_env and update calls and documentation. --- .../clp-package-utils/clp_package_utils/controller.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index a447bf814e..3c833d562e 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -87,9 +87,10 @@ def stop(self): pass @abstractmethod - def _provision(self) -> EnvVarsDict: + def _set_up_env(self) -> EnvVarsDict: """ - Prepares all components with orchestrator-specific logic. + Sets up all components for the orchestrator by preparing environment variables, directories, + and configuration files. :return: Dictionary of environment variables to be used by the orchestrator. """ @@ -471,7 +472,7 @@ def start(self): 3. Running `docker compose up -d`. """ check_docker_dependencies(should_compose_run=False, project_name=self._project_name) - self._provision() + self._set_up_env() deployment_type = self.clp_config.get_deployment_type() logger.info(f"Starting CLP using Docker Compose ({deployment_type})...") @@ -519,7 +520,7 @@ def _get_num_workers() -> int: """ return multiprocessing.cpu_count() // 2 - def _provision(self): + def _set_up_env(self): """ Provisions all CLP components for Docker Compose by: - Generating container-specific config. From 7beb19998dc502b376e7b94a2e014bb98c32135a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:46:32 -0400 Subject: [PATCH 113/238] =?UTF-8?q?docs(clp-package-utils):=20Update=20doc?= =?UTF-8?q?strings=20to=20use=20=E2=80=9Csets=20up=E2=80=9D=20phrasing.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clp_package_utils/controller.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 3c833d562e..e5749b50f5 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -75,7 +75,7 @@ def __init__(self, clp_config: CLPConfig): @abstractmethod def start(self): """ - Starts the provisioned components with orchestrator-specific logic. + Starts the set-up components with orchestrator-specific logic. """ pass @@ -98,7 +98,7 @@ def _set_up_env(self) -> EnvVarsDict: def _set_up_env_for_database(self) -> EnvVarsDict: """ - Prepares environment variables and directories for the database component. + Sets up environment variables and directories for the database component. :return: Dictionary of component-related environment variables. """ @@ -129,7 +129,7 @@ def _set_up_env_for_database(self) -> EnvVarsDict: def _set_up_env_for_queue(self) -> EnvVarsDict: """ - Prepares environment variables and directories for the message queue component. + Sets up environment variables and directories for the message queue component. :return: Dictionary of component-related environment variables. """ @@ -151,7 +151,7 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: def _set_up_env_for_redis(self) -> EnvVarsDict: """ - Prepares environment variables and directories for the Redis component. + Sets up environment variables and directories for the Redis component. :return: Dictionary of component-related environment variables. """ @@ -181,7 +181,7 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: def _set_up_env_for_results_cache(self) -> EnvVarsDict: """ - Prepares environment variables and directories for the results cache (MongoDB) component. + Sets up environment variables and directories for the results cache (MongoDB) component. :return: Dictionary of component-related environment variables. """ @@ -208,7 +208,7 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: """ - Prepares environment variables and files for the compression scheduler component. + Sets up environment variables and files for the compression scheduler component. :return: Dictionary of component-related environment variables. """ @@ -225,7 +225,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: """ - Prepares environment variables and files for the query scheduler component. + Sets up environment variables and files for the query scheduler component. :return: Dictionary of component-related environment variables. """ @@ -242,7 +242,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: """ - Prepares environment variables for the compression worker component. + Sets up environment variables for the compression worker component. :param num_workers: Number of worker processes to run. :return: Dictionary of compression worker-related environment variables. @@ -261,7 +261,7 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: """ - Prepares environment variables for the query worker component. + Sets up environment variables for the query worker component. :param num_workers: Number of worker processes to run. :return: Dictionary of component-related environment variables. @@ -280,7 +280,7 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: """ - Prepares environment variables for the reducer component. + Sets up environment variables for the reducer component. :param num_workers: Number of worker processes to run. :return: Dictionary of component-related environment variables. @@ -300,7 +300,7 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: """ - Prepares environment variables and settings for the Web UI component. + Sets up environment variables and settings for the Web UI component. :param container_clp_config: CLP configuration inside the containers. :return: Dictionary of component-related environment variables. @@ -402,7 +402,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: """ - Prepares environment variables for the garbage collector component. + Sets up environment variables for the garbage collector component. :return: Dictionary of component-related environment variables. """ @@ -468,7 +468,7 @@ def start(self): """ Deploys CLP components using Docker Compose by: 1. Checking Docker dependencies. - 2. Provisioning environment variables and configuration. + 2. Setting up environment variables and configuration. 3. Running `docker compose up -d`. """ check_docker_dependencies(should_compose_run=False, project_name=self._project_name) @@ -522,7 +522,7 @@ def _get_num_workers() -> int: def _set_up_env(self): """ - Provisions all CLP components for Docker Compose by: + Sets up all CLP components for Docker Compose by: - Generating container-specific config. - Preparing environment variables for all components. - Writing environment variables to `.env`. From 3507a1bcc9740b9727ef8d58c5df47b11280a643 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:08:44 -0400 Subject: [PATCH 114/238] docs(clp-package-utils): Standardize return docstrings; Remove return type hint from _set_up_env(). --- .../clp_package_utils/controller.py | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index e5749b50f5..98a1bed560 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -87,12 +87,10 @@ def stop(self): pass @abstractmethod - def _set_up_env(self) -> EnvVarsDict: + def _set_up_env(self): """ Sets up all components for the orchestrator by preparing environment variables, directories, and configuration files. - - :return: Dictionary of environment variables to be used by the orchestrator. """ pass @@ -100,7 +98,7 @@ def _set_up_env_for_database(self) -> EnvVarsDict: """ Sets up environment variables and directories for the database component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = DB_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -131,7 +129,7 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: """ Sets up environment variables and directories for the message queue component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = QUEUE_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -153,7 +151,7 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: """ Sets up environment variables and directories for the Redis component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = REDIS_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -183,7 +181,7 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: """ Sets up environment variables and directories for the results cache (MongoDB) component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = RESULTS_CACHE_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -210,7 +208,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: """ Sets up environment variables and files for the compression scheduler component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -227,7 +225,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: """ Sets up environment variables and files for the query scheduler component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -245,7 +243,7 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: Sets up environment variables for the compression worker component. :param num_workers: Number of worker processes to run. - :return: Dictionary of compression worker-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = COMPRESSION_WORKER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -264,7 +262,7 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: Sets up environment variables for the query worker component. :param num_workers: Number of worker processes to run. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = QUERY_WORKER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -283,7 +281,7 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: Sets up environment variables for the reducer component. :param num_workers: Number of worker processes to run. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = REDUCER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -303,7 +301,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: Sets up environment variables and settings for the Web UI component. :param container_clp_config: CLP configuration inside the containers. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = WEBUI_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -404,7 +402,7 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: """ Sets up environment variables for the garbage collector component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = GARBAGE_COLLECTOR_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -526,8 +524,6 @@ def _set_up_env(self): - Generating container-specific config. - Preparing environment variables for all components. - Writing environment variables to `.env`. - - :return: Dictionary of all environment variables. """ container_clp_config = generate_docker_compose_container_config(self.clp_config) num_workers = self._get_num_workers() From eb2754a84c3d99571846f06bd6e1271073ea9fdc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:10:00 -0400 Subject: [PATCH 115/238] refactor(clp-package-utils): Add spacing before log and data directory setup for readability. --- .../clp-package-utils/clp_package_utils/controller.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 98a1bed560..47d4d2aec1 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -107,6 +107,7 @@ def _set_up_env_for_database(self) -> EnvVarsDict: data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name validate_db_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) @@ -136,6 +137,7 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: logs_dir = self.clp_config.logs_directory / component_name validate_queue_config(self.clp_config, logs_dir) + logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(logs_dir) @@ -160,6 +162,7 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: logs_dir = self.clp_config.logs_directory / component_name data_dir = self.clp_config.data_directory / component_name validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) @@ -190,6 +193,7 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name validate_results_cache_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) @@ -249,6 +253,7 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) return { @@ -268,6 +273,7 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) return { @@ -287,6 +293,7 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) return { @@ -408,6 +415,7 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) return {"CLP_GC_LOGGING_LEVEL": self.clp_config.garbage_collector.logging_level} From cae66df46e60c54bf38571f7a0978ef51a047569 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:22:47 -0400 Subject: [PATCH 116/238] refactor(clp-package-utils): Move logs directory assignment after data directory assignment --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 47d4d2aec1..cbaad687e9 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -159,8 +159,8 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") conf_file = self._conf_dir / "redis" / "redis.conf" - logs_dir = self.clp_config.logs_directory / component_name data_dir = self.clp_config.data_directory / component_name + logs_dir = self.clp_config.logs_directory / component_name validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) From dffc2f2a648064b7407b9a4da0169d42f370c522 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:23:44 -0400 Subject: [PATCH 117/238] refactor(clp-package-utils): Rename logs_file -> log_file --- .../clp_package_utils/controller.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index cbaad687e9..2aea50f55c 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -217,12 +217,12 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_file = self.clp_config.logs_directory / f"{component_name}.log" - logs_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) + log_file = self.clp_config.logs_directory / f"{component_name}.log" + log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) return { "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self.clp_config.compression_scheduler.logging_level, - "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), + "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(log_file), } def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: @@ -234,12 +234,12 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_file = self.clp_config.logs_directory / f"{component_name}.log" - logs_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) + log_file = self.clp_config.logs_directory / f"{component_name}.log" + log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) return { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self.clp_config.query_scheduler.logging_level, - "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), + "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(log_file), } def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: From c0ef4ff24b5ecf12f863729b625a013c5d1a1e29 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:27:59 -0400 Subject: [PATCH 118/238] refactor(clp-package-utils): Rename LOGS_FILE variables to LOG_FILE in Docker compose files and controller. --- components/clp-package-utils/clp_package_utils/controller.py | 4 ++-- tools/deployment/package/docker-compose.base.yaml | 2 +- tools/deployment/package/docker-compose.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 2aea50f55c..d87e29b3a1 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -222,7 +222,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: return { "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self.clp_config.compression_scheduler.logging_level, - "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(log_file), + "CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST": str(log_file), } def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: @@ -239,7 +239,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: return { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self.clp_config.query_scheduler.logging_level, - "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(log_file), + "CLP_QUERY_SCHEDULER_LOG_FILE_HOST": str(log_file), } def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 11d00dbd9d..b70b06e1c4 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -167,7 +167,7 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}\ + - "${CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST:-./var/log/compression_scheduler.log}\ :/var/log/compression_scheduler.log" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index e793d08dfe..a58310cb3f 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -31,7 +31,7 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_QUERY_SCHEDULER_LOGS_FILE_HOST:-./var/log/query_scheduler.log}:\ + - "${CLP_QUERY_SCHEDULER_LOG_FILE_HOST:-./var/log/query_scheduler.log}:\ /var/log/query_scheduler.log" depends_on: db-table-creator: From 95404d0f72dbcf0e025989d201401f3b6d15cb99 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:29:55 -0400 Subject: [PATCH 119/238] refactor(clp-package-utils): Remove redundant flush after writing instance id. --- components/clp-package-utils/clp_package_utils/controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index d87e29b3a1..0f31a423c6 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -590,7 +590,6 @@ def get_or_create_instance_id(clp_config: CLPConfig): instance_id = str(uuid.uuid4())[-4:] with open(instance_id_file_path, "w") as f: f.write(instance_id) - f.flush() return instance_id From e1db192b53477422b5c1db429458d6b6a8dc8f1a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:32:14 -0400 Subject: [PATCH 120/238] Clarify resolving to IPv4 in the docstring - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 0f31a423c6..51b9af2b6d 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -625,7 +625,7 @@ def _chown_recursively( def _get_ip_from_hostname(hostname: str) -> str: """ - Resolves a hostname to an IP address. + Resolves a hostname to an IPv4 IP address. :param hostname: The hostname to resolve. :return: The resolved IP address. From 630329b71fd5bca9f185c48d0c8dd0e0918e751c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:33:33 -0400 Subject: [PATCH 121/238] Remove redundant param description in `_get_ip_from_hostname()` - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 51b9af2b6d..3b30875844 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -627,7 +627,7 @@ def _get_ip_from_hostname(hostname: str) -> str: """ Resolves a hostname to an IPv4 IP address. - :param hostname: The hostname to resolve. + :param hostname: :return: The resolved IP address. """ return socket.gethostbyname(hostname) From d9da76384a041f3b5b7644d40dbd27dfb0edaee0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:48:40 -0400 Subject: [PATCH 122/238] refactor(clp-package-utils): Rename db logging config variable and env placeholder for clarity. --- .../clp-package-utils/clp_package_utils/controller.py | 6 +++--- tools/deployment/package/docker-compose.base.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 3b30875844..136ae09f04 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -103,17 +103,17 @@ def _set_up_env_for_database(self) -> EnvVarsDict: component_name = DB_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - conf_file = self._conf_dir / "mysql" / "conf.d" / "logging.cnf" + conf_logging_file = self._conf_dir / "mysql" / "conf.d" / "logging.cnf" data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name - validate_db_config(self.clp_config, conf_file, data_dir, logs_dir) + validate_db_config(self.clp_config, conf_logging_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) return { - "CLP_DB_CONF_FILE_HOST": str(conf_file), + "CLP_DB_CONF_LOGGING_FILE_HOST": str(conf_logging_file), "CLP_DB_DATA_DIR_HOST": str(data_dir), "CLP_DB_LOGS_DIR_HOST": str(logs_dir), "CLP_DB_HOST": _get_ip_from_hostname(self.clp_config.database.host), diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index b70b06e1c4..fa545b68d3 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -34,7 +34,7 @@ services: ports: - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" volumes: - - "${CLP_DB_CONF_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" + - "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" healthcheck: From c2005028e22d7d9c2f54250ff891f6e44d82a7b1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:55:56 -0400 Subject: [PATCH 123/238] use long form `:readonly` instead of short form `:ro` in binding mount specifications --- .../package/docker-compose.base.yaml | 29 ++++++++++--------- tools/deployment/package/docker-compose.yaml | 8 ++--- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index fa545b68d3..956aab4649 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -34,7 +34,8 @@ services: ports: - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" volumes: - - "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" + - "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}\ +:/etc/mysql/conf.d/logging.cnf:readonly" - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" healthcheck: @@ -96,7 +97,8 @@ services: ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" volumes: - - "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}:/usr/local/etc/redis/redis.conf:ro" + - "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}:/usr/local/etc/redis/redis.conf\ +:readonly" - "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}:/data" - "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}:/var/log/redis" healthcheck: @@ -123,7 +125,8 @@ services: ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" volumes: - - "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" + - "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf\ +:readonly" - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" healthcheck: @@ -166,11 +169,11 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - "${CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST:-./var/log/compression_scheduler.log}\ :/var/log/compression_scheduler.log" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "/:/mnt/logs:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" + - "/:/mnt/logs:readonly" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -203,12 +206,12 @@ services: volumes: - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ /var/log/compression_worker" - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "/:/mnt/logs:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" + - "/:/mnt/logs:readonly" command: [ "python3", "-u", @@ -236,11 +239,11 @@ services: ports: - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" - - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" + - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:readonly" - "./var/www/webui/server/dist/settings.json\ -:/opt/clp/var/www/webui/server/dist/settings.json:ro" +:/opt/clp/var/www/webui/server/dist/settings.json:readonly" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -270,7 +273,7 @@ services: CLP_LOGS_DIR: "/var/log/garbage_collector" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" depends_on: db-table-creator: diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index a58310cb3f..45bfc222bd 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -30,7 +30,7 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - "${CLP_QUERY_SCHEDULER_LOG_FILE_HOST:-./var/log/query_scheduler.log}:\ /var/log/query_scheduler.log" depends_on: @@ -73,8 +73,8 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - "${CLP_QUERY_WORKER_LOGS_DIR_HOST:-./var/log/query_worker}:/var/log/query_worker" - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" @@ -100,7 +100,7 @@ services: CLP_LOGS_DIR: "/var/log/reducer" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - "${CLP_REDUCER_LOGS_DIR_HOST:-./var/log/reducer}:/var/log/reducer" depends_on: query-scheduler: From 5751ddf58505ffbf690ea9888b6995549f4cdc6a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 19:24:03 -0400 Subject: [PATCH 124/238] revert unintentional change --- docs/src/user-docs/guides-overview.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/user-docs/guides-overview.md b/docs/src/user-docs/guides-overview.md index 62f01a9835..60d7bbacd1 100644 --- a/docs/src/user-docs/guides-overview.md +++ b/docs/src/user-docs/guides-overview.md @@ -32,3 +32,4 @@ Using Presto with CLP ^^^ How to use Presto to query compressed logs in CLP. ::: +:::: From 3e9734ed8e079c8e1f3ab637579bbd659702db1e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 19:29:56 -0400 Subject: [PATCH 125/238] chore(deployment): Add local logging driver to service defaults. --- tools/deployment/package/docker-compose.base.yaml | 2 ++ tools/deployment/package/docker-compose.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 956aab4649..a120906d1c 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -3,6 +3,8 @@ name: "clp-package-base" # Common service defaults. x-service-defaults: &service_defaults image: "${CLP_PACKAGE_CONTAINER:-clp-package}" + logging: + driver: "local" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 45bfc222bd..70b062458f 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -5,6 +5,8 @@ include: ["docker-compose.base.yaml"] # Common service defaults. x-service-defaults: &service_defaults image: "${CLP_PACKAGE_CONTAINER:-clp-package}" + logging: + driver: "local" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" From ab64f4aca35473fcc54287e19b0321098fd6a526 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 20:12:33 -0400 Subject: [PATCH 126/238] Improve docs - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- docs/src/user-docs/guides-multi-node.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/user-docs/guides-multi-node.md b/docs/src/user-docs/guides-multi-node.md index 9a56f4c631..52f206f7af 100644 --- a/docs/src/user-docs/guides-multi-node.md +++ b/docs/src/user-docs/guides-multi-node.md @@ -3,8 +3,8 @@ A multi-node deployment allows you to run CLP across a distributed set of hosts. :::{warning} -CLP now uses Docker Compose for orchestration and support for multi-node deployments is -temporarily removed. Please contact us if you need immediate support for multi-node deployments, or +CLP now uses Docker Compose for orchestration and support for multi-node deployments has been +removed temporarily. Please contact us if you need immediate support for multi-node deployments, or stay tuned for future updates on Kubernetes Helm support. ::: From 2d9906f8723c7dace88f87ff184c72312d97475b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 20:48:20 -0400 Subject: [PATCH 127/238] chore(deployment): Adjust healthcheck defaults (start interval, period, timeout). --- tools/deployment/package/docker-compose.base.yaml | 6 +++--- tools/deployment/package/docker-compose.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index a120906d1c..a89b0bf4b9 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -12,9 +12,9 @@ x-service-defaults: &service_defaults x-healthcheck-defaults: &healthcheck_defaults interval: "30s" retries: 3 - start_interval: "1s" - start_period: "10s" - timeout: "10s" + start_interval: "2s" + start_period: "60s" + timeout: "2s" secrets: CLP_DB_PASS_FILE: diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 70b062458f..4aae85e367 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -14,9 +14,9 @@ x-service-defaults: &service_defaults x-healthcheck-defaults: &healthcheck_defaults interval: "30s" retries: 3 - start_interval: "1s" - start_period: "10s" - timeout: "10s" + start_interval: "2s" + start_period: "60s" + timeout: "2s" services: query-scheduler: From 3635883b314789b310b51c9fdde0519cb3564d45 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 22:31:19 -0400 Subject: [PATCH 128/238] refactor(deployment): Use long form volume definitions and use bind objects with anchors. --- .../package/docker-compose.base.yaml | 123 +++++++++++++----- tools/deployment/package/docker-compose.yaml | 46 +++++-- 2 files changed, 124 insertions(+), 45 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index a89b0bf4b9..1ff53e5b5f 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -16,6 +16,24 @@ x-healthcheck-defaults: &healthcheck_defaults start_period: "60s" timeout: "2s" +# Common volume definitions. +x-volume-definitions: + aws-config-readonly: &volume_aws_config_readonly + type: "bind" + source: "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}" + target: "/.aws" + read_only: true + clp-config-readonly: &volume_clp_config_readonly + type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" + target: "/etc/clp-config.yml" + read_only: true + logs-input-readonly: &volume_root_logs_readonly + type: "bind" + source: "/" + target: "/mnt/logs" + read_only: true + secrets: CLP_DB_PASS_FILE: environment: "CLP_DB_PASS" @@ -36,10 +54,16 @@ services: ports: - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" volumes: - - "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}\ -:/etc/mysql/conf.d/logging.cnf:readonly" - - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" - - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" + - type: "bind" + source: "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}" + target: "/etc/mysql/conf.d/logging.cnf" + read_only: true + - type: "bind" + source: "${CLP_DB_DATA_DIR_HOST:-./var/data/database}" + target: "/var/lib/mysql" + - type: "bind" + source: "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}" + target: "/var/log/mysql" healthcheck: <<: *healthcheck_defaults test: [ @@ -59,7 +83,7 @@ services: CLP_DB_USER: "${CLP_DB_USER}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" + - *volume_clp_config_readonly depends_on: database: condition: "service_healthy" @@ -83,7 +107,9 @@ services: ports: - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" volumes: - - "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}:/var/log/rabbitmq" + - type: "bind" + source: "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}" + target: "/var/log/rabbitmq" healthcheck: <<: *healthcheck_defaults test: [ @@ -99,10 +125,16 @@ services: ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" volumes: - - "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}:/usr/local/etc/redis/redis.conf\ -:readonly" - - "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}:/data" - - "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}:/var/log/redis" + - type: "bind" + source: "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}" + target: "/usr/local/etc/redis/redis.conf" + read_only: true + - type: "bind" + source: "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}" + target: "/data" + - type: "bind" + source: "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}" + target: "/var/log/redis" healthcheck: <<: *healthcheck_defaults test: [ @@ -127,10 +159,16 @@ services: ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" volumes: - - "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf\ -:readonly" - - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" - - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" + - type: "bind" + source: "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}" + target: "/etc/mongo/mongod.conf" + read_only: true + - type: "bind" + source: "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}" + target: "/data/db" + - type: "bind" + source: "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}" + target: "/var/log/mongodb" healthcheck: <<: *healthcheck_defaults test: [ @@ -171,11 +209,12 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - - "${CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST:-./var/log/compression_scheduler.log}\ -:/var/log/compression_scheduler.log" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - - "/:/mnt/logs:readonly" + - *volume_aws_config_readonly + - *volume_clp_config_readonly + - *volume_root_logs_readonly + - type: "bind" + source: "${CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST:-./var/log/compression_scheduler.log}" + target: "/var/log/compression_scheduler.log" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -206,14 +245,21 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ -/var/log/compression_worker" - - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - - "/:/mnt/logs:readonly" + - *volume_aws_config_readonly + - *volume_clp_config_readonly + - *volume_root_logs_readonly + - type: "bind" + source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" + target: "/var/data/archives" + - type: "bind" + source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}" + target: "/var/data/staged-archives" + - type: "bind" + source: "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}" + target: "/var/log/compression_worker" + - type: "bind" + source: "${CLP_DATA_DIR_HOST:-./var/data}" + target: "/var/data" command: [ "python3", "-u", @@ -241,11 +287,18 @@ services: ports: - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" - - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:readonly" - - "./var/www/webui/server/dist/settings.json\ -:/opt/clp/var/www/webui/server/dist/settings.json:readonly" + - *volume_aws_config_readonly + - type: "bind" + source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}" + target: "/var/data/streams" + - type: "bind" + source: "./var/www/webui/client/settings.json" + target: "/opt/clp/var/www/webui/client/settings.json" + read_only: true + - type: "bind" + source: "./var/www/webui/server/dist/settings.json" + target: "/opt/clp/var/www/webui/server/dist/settings.json" + read_only: true depends_on: db-table-creator: condition: "service_completed_successfully" @@ -275,8 +328,10 @@ services: CLP_LOGS_DIR: "/var/log/garbage_collector" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" + - *volume_clp_config_readonly + - type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector" + target: "/var/log/garbage_collector" depends_on: db-table-creator: condition: "service_completed_successfully" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 4aae85e367..acd54ed415 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -18,6 +18,19 @@ x-healthcheck-defaults: &healthcheck_defaults start_period: "60s" timeout: "2s" +# Common volume definitions. +x-volume-definitions: + aws-config-readonly: &volume_aws_config_readonly + type: "bind" + source: "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}" + target: "/.aws" + read_only: true + clp-config-readonly: &volume_clp_config_readonly + type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" + target: "/etc/clp-config.yml" + read_only: true + services: query-scheduler: <<: *service_defaults @@ -32,9 +45,10 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - - "${CLP_QUERY_SCHEDULER_LOG_FILE_HOST:-./var/log/query_scheduler.log}:\ -/var/log/query_scheduler.log" + - *volume_clp_config_readonly + - type: "bind" + source: "${CLP_QUERY_SCHEDULER_LOG_FILE_HOST:-./var/log/query_scheduler.log}" + target: "/var/log/query_scheduler.log" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -74,12 +88,20 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - - "${CLP_QUERY_WORKER_LOGS_DIR_HOST:-./var/log/query_worker}:/var/log/query_worker" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" + - *volume_aws_config_readonly + - *volume_clp_config_readonly + - type: "bind" + source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" + target: "/var/data/archives" + - type: "bind" + source: "${CLP_QUERY_WORKER_LOGS_DIR_HOST:-./var/log/query_worker}" + target: "/var/log/query_worker" + - type: "bind" + source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}" + target: "/var/data/staged-streams" + - type: "bind" + source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}" + target: "/var/data/streams" command: [ "python3", "-u", @@ -102,8 +124,10 @@ services: CLP_LOGS_DIR: "/var/log/reducer" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - - "${CLP_REDUCER_LOGS_DIR_HOST:-./var/log/reducer}:/var/log/reducer" + - *volume_clp_config_readonly + - type: "bind" + source: "${CLP_REDUCER_LOGS_DIR_HOST:-./var/log/reducer}" + target: "/var/log/reducer" depends_on: query-scheduler: condition: "service_healthy" From 5e0d0d2311813fa5c01970812a6a9bd6ebc6ce89 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 23:13:59 -0400 Subject: [PATCH 129/238] refactor(deployment): Use long form port definitions; fix CLP_RESULTS_CACHE_PORT fallback 6379 -> 27017. --- .../package/docker-compose.base.yaml | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 1ff53e5b5f..c6e0730a9b 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -52,7 +52,9 @@ services: secrets: - "CLP_DB_PASS_FILE" ports: - - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" + - host_ip: "${CLP_DB_HOST:-127.0.0.1}" + target: 3306 + published: "${CLP_DB_PORT:-3306}" volumes: - type: "bind" source: "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}" @@ -105,7 +107,9 @@ services: RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" ports: - - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" + - host_ip: "${CLP_QUEUE_HOST:-127.0.0.1}" + target: 5672 + published: "${CLP_QUEUE_PORT:-5672}" volumes: - type: "bind" source: "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}" @@ -123,7 +127,9 @@ services: hostname: "redis" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" + - host_ip: "${CLP_REDIS_HOST:-127.0.0.1}" + target: 6379 + published: "${CLP_REDIS_PORT:-6379}" volumes: - type: "bind" source: "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}" @@ -157,7 +163,9 @@ services: hostname: "results_cache" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" + - host_ip: "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}" + target: 27017 + published: "${CLP_RESULTS_CACHE_PORT:-27017}" volumes: - type: "bind" source: "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}" @@ -285,7 +293,9 @@ services: PORT: "4000" RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" ports: - - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" + - host_ip: "${CLP_WEBUI_HOST:-127.0.0.1}" + target: 4000 + published: "${CLP_WEBUI_PORT:-4000}" volumes: - *volume_aws_config_readonly - type: "bind" From e771800aed92323320a5194cae23516f12bec62b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 23:20:50 -0400 Subject: [PATCH 130/238] refactor(deployment): Reorder port definitions to long form order. --- tools/deployment/package/docker-compose.base.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index c6e0730a9b..7461ce3e6c 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -53,8 +53,8 @@ services: - "CLP_DB_PASS_FILE" ports: - host_ip: "${CLP_DB_HOST:-127.0.0.1}" - target: 3306 published: "${CLP_DB_PORT:-3306}" + target: 3306 volumes: - type: "bind" source: "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}" @@ -108,8 +108,8 @@ services: RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" ports: - host_ip: "${CLP_QUEUE_HOST:-127.0.0.1}" - target: 5672 published: "${CLP_QUEUE_PORT:-5672}" + target: 5672 volumes: - type: "bind" source: "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}" @@ -128,8 +128,8 @@ services: user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - host_ip: "${CLP_REDIS_HOST:-127.0.0.1}" - target: 6379 published: "${CLP_REDIS_PORT:-6379}" + target: 6379 volumes: - type: "bind" source: "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}" @@ -164,8 +164,8 @@ services: user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - host_ip: "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}" - target: 27017 published: "${CLP_RESULTS_CACHE_PORT:-27017}" + target: 27017 volumes: - type: "bind" source: "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}" @@ -294,8 +294,8 @@ services: RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" ports: - host_ip: "${CLP_WEBUI_HOST:-127.0.0.1}" - target: 4000 published: "${CLP_WEBUI_PORT:-4000}" + target: 4000 volumes: - *volume_aws_config_readonly - type: "bind" From 6e878cf3a6aa58baf5286b145c8365eb3c256112 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 23:47:15 -0400 Subject: [PATCH 131/238] refactor(clp-package-utils): Replace deprecated Pydantic (V2) copy() with model_copy() --- components/clp-package-utils/clp_package_utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 9fa7f946b1..c428e0b351 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -313,7 +313,7 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig :param clp_config: :return: The container config and the mounts. """ - container_clp_config = clp_config.copy(deep=True) + container_clp_config = clp_config.model_copy(deep=True) container_clp_config.transform_for_container() return container_clp_config From f2b096765f82c4c77e5e05b7eb2fdc8c2a568ecf Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 7 Oct 2025 00:15:22 -0400 Subject: [PATCH 132/238] feat(deployment): Add support for configurable logs input directory mapping via env vars. --- .../clp-package-utils/clp_package_utils/controller.py | 6 +++++- components/clp-py-utils/clp_py_utils/clp_config.py | 4 +++- tools/deployment/package/docker-compose.base.yaml | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 136ae09f04..0b3807a3dd 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -546,9 +546,13 @@ def _set_up_env(self): ), # Package container "CLP_PACKAGE_CONTAINER": self.clp_config.container_image_ref, - # Global paths + # Runtime data directories "CLP_DATA_DIR_HOST": str(self.clp_config.data_directory), "CLP_LOGS_DIR_HOST": str(self.clp_config.logs_directory), + # Input directories + "CLP_LOGS_INPUT_DIR_HOST": str(self.clp_config.logs_input.directory), + "CLP_LOGS_INPUT_DIR_CONTAINER": str(container_clp_config.logs_input.directory), + # Output directories "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self.clp_config.archive_output.get_directory()), "CLP_STREAM_OUTPUT_DIR_HOST": str(self.clp_config.stream_output.get_directory()), # AWS credentials diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index b2eb995ea2..29fc9c6310 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -685,7 +685,9 @@ class FsIngestionConfig(FsStorage): directory: pathlib.Path = pathlib.Path("/") def transform_for_container(self): - self.directory = CONTAINER_INPUT_LOGS_ROOT_DIR + input_logs_dir = self.directory.resolve() + self.directory = (CONTAINER_INPUT_LOGS_ROOT_DIR / + input_logs_dir.relative_to(input_logs_dir.anchor)) class ArchiveFsStorage(FsStorage): diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 7461ce3e6c..52a4d8eeec 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -30,8 +30,8 @@ x-volume-definitions: read_only: true logs-input-readonly: &volume_root_logs_readonly type: "bind" - source: "/" - target: "/mnt/logs" + source: "${CLP_LOGS_INPUT_DIR_HOST:-/}" + target: "${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" read_only: true secrets: From 01558ba3dc6412342007b512391d109ea812f26a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 7 Oct 2025 03:48:23 -0400 Subject: [PATCH 133/238] lint --- components/clp-py-utils/clp_py_utils/clp_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 29fc9c6310..43d1695e60 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -686,8 +686,9 @@ class FsIngestionConfig(FsStorage): def transform_for_container(self): input_logs_dir = self.directory.resolve() - self.directory = (CONTAINER_INPUT_LOGS_ROOT_DIR / - input_logs_dir.relative_to(input_logs_dir.anchor)) + self.directory = CONTAINER_INPUT_LOGS_ROOT_DIR / input_logs_dir.relative_to( + input_logs_dir.anchor + ) class ArchiveFsStorage(FsStorage): From 8a3985ba496e7c008e263297d554dd71aa5f28d0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 7 Oct 2025 04:05:20 -0400 Subject: [PATCH 134/238] refactor(deployment): Remove secrets usage; replace with environment variables. --- tools/deployment/package/docker-compose.base.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 52a4d8eeec..6c27235240 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -34,10 +34,6 @@ x-volume-definitions: target: "${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" read_only: true -secrets: - CLP_DB_PASS_FILE: - environment: "CLP_DB_PASS" - services: database: <<: *service_defaults @@ -46,11 +42,9 @@ services: user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" environment: MYSQL_DATABASE: "${CLP_DB_NAME}" - MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" - MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_PASSWORD: "${CLP_DB_PASS}" + MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS}" MYSQL_USER: "${CLP_DB_USER}" - secrets: - - "CLP_DB_PASS_FILE" ports: - host_ip: "${CLP_DB_HOST:-127.0.0.1}" published: "${CLP_DB_PORT:-3306}" From c691735e4ab28c7068ad65bf5738472e694242db Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 7 Oct 2025 04:18:42 -0400 Subject: [PATCH 135/238] docs(design): Add health check defaults. --- docs/src/dev-docs/design-docker-compose.md | 42 ++++++++++++++++------ 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index e365decd63..eb14b23e8a 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -4,8 +4,8 @@ This document explains the technical details of CLP's Docker Compose implementat ## Overview -The Docker Compose implementation follows a controller architecture with a `BaseController` abstract class and a -`DockerComposeController` implementation. +The Docker Compose implementation follows a controller architecture with a `BaseController` abstract +class and a `DockerComposeController` implementation. ## Architecture @@ -20,14 +20,16 @@ The orchestration uses a controller pattern: The controller performs these initialization steps: -1. **Provisioning**: Provisions all components and generates component specific configuration variables. -2. **Configuration Transformation**: The `transform_for_container()` method in `CLPConfig` adapts configurations for - containerized environments +1. **Provisioning**: Provisions all components and generates component specific configuration + variables. +2. **Configuration Transformation**: The `transform_for_container()` method in `CLPConfig` adapts + configurations for containerized environments 3. **Environment Generation**: Creates a `.env` file with necessary Docker Compose variables ### Configuration Transformation -The `transform_for_container()` method in the `CLPConfig` class and related component classes adapts the configuration for containerized environments by: +The `transform_for_container()` method in the `CLPConfig` class and related component classes +adapts the configuration for containerized environments by: 1. Converting host paths to container paths 2. Updating service hostnames to match Docker Compose service names @@ -35,7 +37,8 @@ The `transform_for_container()` method in the `CLPConfig` class and related comp ### Environment Variables -The controller generates a comprehensive set of environment variables that are written to a `.env` file, including: +The controller generates a comprehensive set of environment variables that are written to a `.env` +file, including: * Component-specific settings (ports, logging levels, concurrency) * Credentials for database, queue, and Redis services @@ -58,8 +61,8 @@ CLP supports two deployment types determined by the `package.query_engine` confi The Docker Compose setup uses two files: -* `docker-compose.base.yaml`: Defines base services for all deployment types, excluding Celery scheduler and worker - components to allow separate Presto [integration][presto-integration]. +* `docker-compose.base.yaml`: Defines base services for all deployment types, excluding Celery + scheduler and worker components to allow separate Presto [integration][presto-integration]. * `docker-compose.yaml`: Extends the base file with additional services for complete deployments Each file defines services with: @@ -70,6 +73,22 @@ Each file defines services with: * Network configuration * User permissions +### Health check defaults + +Below are the default health check settings and the rationale for each: + +* `interval: 30s` — default probe interval in steady state. Avoid setting this too low to avoid + excessive resource usage. +* `timeout: 2s` — no remote communication is expected, so a short timeout is sufficient. +* `retries: 3` + - a service is deemed unhealthy if it does not respond in ~(30s * 3) = 90s since it is in the + steady state. + - a service is deemed unhealthy if it does not respond within ~(60s + 90s) since it is started. +* `start_interval: 2s` — A short interval allows the service to become healthy quickly once it + is ready, allowing other services which depend on it to start. +* `start_period: 60s` — the first minute of startup ignores failures, effectively granting around 30 + fast attempts to become healthy before retries start counting. + ## Service architecture The Docker Compose setup includes the following services: @@ -140,7 +159,8 @@ graph LR ### Services overview -The CLP package is composed of several service components. The tables below list the services and their functions. +The CLP package is composed of several service components. The tables below list the services and +their functions. :::{table} Services :align: left @@ -192,4 +212,4 @@ If you encounter issues with the Docker Compose deployment: docker compose config ``` -[presto-integration]: ../user-docs/guides-using-presto.md \ No newline at end of file +[presto-integration]: ../user-docs/guides-using-presto.md From 787c7d77b0bf9d21e99d874335b807d85c8538e2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 7 Oct 2025 04:27:37 -0400 Subject: [PATCH 136/238] refactor(clp-package-utils): Improve error handling for config loading and directory creation. --- .../clp_package_utils/scripts/start_clp.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 368612530f..a9ff5b596e 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -34,8 +34,8 @@ def main(argv): parsed_args = args_parser.parse_args(argv[1:]) - # Validate and load config file try: + # Validate and load config file. config_file_path = pathlib.Path(parsed_args.config) clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) @@ -49,14 +49,18 @@ def main(argv): clp_config.validate_data_dir() clp_config.validate_logs_dir() clp_config.validate_aws_config_dir() + except: + logger.exception("Failed to load config.") + return -1 - # Create necessary directories + try: + # Create necessary directories. clp_config.data_directory.mkdir(parents=True, exist_ok=True) clp_config.logs_directory.mkdir(parents=True, exist_ok=True) clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) except: - logger.exception("Failed to load config.") + logger.exception("Failed to create necessary directories.") return -1 instance_id = get_or_create_instance_id(clp_config) From 8b97bc2f857402bf2ea12ae0b39ac91e2fc2e6a3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 9 Oct 2025 01:53:31 -0400 Subject: [PATCH 137/238] refactor(deployment): Remove outdated comment in reducer healthcheck configuration. --- tools/deployment/package/docker-compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index acd54ed415..979f9c314c 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -64,7 +64,6 @@ services: ] healthcheck: <<: *healthcheck_defaults - # FIXME: need to suppressing warnings in the scheduler for reading 0 out of 8 expected bytes test: [ "CMD", "bash", From ccb8f66a993f2f1f65f5b38dfea5ed96e09d1f85 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 9 Oct 2025 14:48:12 -0400 Subject: [PATCH 138/238] refactor(deployment): Remove duplicated docstring from docker-compose.yaml; comment that the other file should be referred --- tools/deployment/package/docker-compose.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 979f9c314c..44ecae5e89 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -2,23 +2,20 @@ name: "clp-package" include: ["docker-compose.base.yaml"] -# Common service defaults. +# Below x-* definitions are duplicated from docker-compose.base.yaml. Refer to that file for +# documentation. x-service-defaults: &service_defaults image: "${CLP_PACKAGE_CONTAINER:-clp-package}" logging: driver: "local" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" - -# Common healthcheck defaults. x-healthcheck-defaults: &healthcheck_defaults interval: "30s" retries: 3 start_interval: "2s" start_period: "60s" timeout: "2s" - -# Common volume definitions. x-volume-definitions: aws-config-readonly: &volume_aws_config_readonly type: "bind" From 40b88a07eefe5236d0d7db5ceed892d8106d3c75 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 9 Oct 2025 15:02:52 -0400 Subject: [PATCH 139/238] docs(design): Remove health check defaults section; add comments to healthcheck defaults in docker-compose.base.yaml --- docs/src/dev-docs/design-docker-compose.md | 16 ---------------- .../deployment/package/docker-compose.base.yaml | 7 +++++++ 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index eb14b23e8a..9bec8bb49b 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -73,22 +73,6 @@ Each file defines services with: * Network configuration * User permissions -### Health check defaults - -Below are the default health check settings and the rationale for each: - -* `interval: 30s` — default probe interval in steady state. Avoid setting this too low to avoid - excessive resource usage. -* `timeout: 2s` — no remote communication is expected, so a short timeout is sufficient. -* `retries: 3` - - a service is deemed unhealthy if it does not respond in ~(30s * 3) = 90s since it is in the - steady state. - - a service is deemed unhealthy if it does not respond within ~(60s + 90s) since it is started. -* `start_interval: 2s` — A short interval allows the service to become healthy quickly once it - is ready, allowing other services which depend on it to start. -* `start_period: 60s` — the first minute of startup ignores failures, effectively granting around 30 - fast attempts to become healthy before retries start counting. - ## Service architecture The Docker Compose setup includes the following services: diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 6c27235240..2c00c58f15 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -10,10 +10,17 @@ x-service-defaults: &service_defaults # Common healthcheck defaults. x-healthcheck-defaults: &healthcheck_defaults + # Avoid lowering to prevent excessive resource usage. interval: "30s" + # Mark unhealthy after 3 failed probes. + # - In steady state, ( + ) × 3 = ~90s before the service is marked unhealthy. + # - From startup, (60s) + ~90s = ~150s before the service is marked unhealthy. retries: 3 + # Frequent checks during startup allow fast transition to healthy. start_interval: "2s" + # Ignore failures for ~15 frequent checks before counting retries. start_period: "60s" + # Short timeout since no remote communication is expected. timeout: "2s" # Common volume definitions. From e58512a7fc42f8a225dce7da945870ea3ab085c8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 9 Oct 2025 22:38:57 -0400 Subject: [PATCH 140/238] docs(quick-start): Add required containerd, Docker CE, CLI, and compose plugin versions to requirements list. --- docs/src/user-docs/quick-start/index.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/src/user-docs/quick-start/index.md b/docs/src/user-docs/quick-start/index.md index cfe7fea552..e46086ed26 100644 --- a/docs/src/user-docs/quick-start/index.md +++ b/docs/src/user-docs/quick-start/index.md @@ -13,6 +13,10 @@ This guide describes the following: To run a CLP release, you'll need: * [Docker](#docker) + * `containerd.io` >= 1.7.18 + * `docker-ce` >= 27.0.3 + * `docker-ce-cli` >= 27.0.3 + * `docker-compose-plugin` >= 2.28.1 * [Python](#python) ### Docker From 37a0a8a6d5e45d9afd7b6fa5be76b9f94e8fd708 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 10 Oct 2025 14:25:28 -0400 Subject: [PATCH 141/238] restore verbose logging option to start_clp script. --- .../clp_package_utils/scripts/start_clp.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index a9ff5b596e..f84ba7c7a3 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -31,9 +31,20 @@ def main(argv): default=str(default_config_file_path), help="CLP package configuration file.", ) + args_parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Enable debug logging.", + ) parsed_args = args_parser.parse_args(argv[1:]) + if parsed_args.verbose: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + try: # Validate and load config file. config_file_path = pathlib.Path(parsed_args.config) From 95d900a07a831a28ae3b23c96bc19040c63410ac Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 14 Oct 2025 19:31:06 -0400 Subject: [PATCH 142/238] refactor(clp-py-utils): Use DEFAULT_PORT for WebUi port configuration. --- components/clp-py-utils/clp_py_utils/clp_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index f20026d154..41a9ee431d 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -575,8 +575,10 @@ def dump_to_primitive_dict(self): class WebUi(BaseModel): + DEFAULT_PORT: ClassVar[int] = 4000 + host: DomainStr = "localhost" - port: Port = 4000 + port: Port = DEFAULT_PORT results_metadata_collection_name: NonEmptyStr = "results-metadata" rate_limit: PositiveInt = 1000 From f6cbc7f97b36767ee069c64d8ec6551431ffe829 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 03:15:21 -0400 Subject: [PATCH 143/238] apply docs suggestions --- .../clp_package_utils/controller.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 0b3807a3dd..6162022266 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -48,7 +48,6 @@ validate_webui_config, ) -# Type alias for environment variables dictionary. EnvVarsDict = Dict[str, str] LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH @@ -63,8 +62,9 @@ class BaseController(ABC): """ - Abstract base controller for preparing and deploying CLP components. Provides common logic for - preparing environment variables, directories, and configuration files for each service. + Base controller for orchestrating CLP components. Derived classes should implement any + orchestrator-specific logic. This class provides common logic for preparing environment + variables, directories, and configuration files for each component. """ def __init__(self, clp_config: CLPConfig): @@ -75,22 +75,22 @@ def __init__(self, clp_config: CLPConfig): @abstractmethod def start(self): """ - Starts the set-up components with orchestrator-specific logic. + Starts the components. """ pass @abstractmethod def stop(self): """ - Stops the deployed components with orchestrator-specific logic. + Stops the components. """ pass @abstractmethod def _set_up_env(self): """ - Sets up all components for the orchestrator by preparing environment variables, directories, - and configuration files. + Sets up all components to run by preparing environment variables, directories, and + configuration files. """ pass @@ -463,7 +463,7 @@ def _update_settings_object( class DockerComposeController(BaseController): """ - Controller for deploying CLP components using Docker Compose. + Controller for orchestrating CLP components using Docker Compose. """ def __init__(self, clp_config: CLPConfig, instance_id: str): @@ -472,16 +472,13 @@ def __init__(self, clp_config: CLPConfig, instance_id: str): def start(self): """ - Deploys CLP components using Docker Compose by: - 1. Checking Docker dependencies. - 2. Setting up environment variables and configuration. - 3. Running `docker compose up -d`. + Starts CLP's components using Docker Compose. """ check_docker_dependencies(should_compose_run=False, project_name=self._project_name) self._set_up_env() deployment_type = self.clp_config.get_deployment_type() - logger.info(f"Starting CLP using Docker Compose ({deployment_type})...") + logger.info(f"Starting CLP using Docker Compose ({deployment_type} deployment)...") cmd = ["docker", "compose", "--project-name", self._project_name] if deployment_type == DeploymentType.BASE: @@ -520,9 +517,8 @@ def stop(self): @staticmethod def _get_num_workers() -> int: """ - Gets the parallelism number for worker components. TODO: Revisit after moving from single-container to multi-container workers. - :return: Number of worker processes. + :return: Number of worker processes to run. """ return multiprocessing.cpu_count() // 2 @@ -558,6 +554,7 @@ def _set_up_env(self): # AWS credentials "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), + # Component-specific environment variables **self._set_up_env_for_database(), **self._set_up_env_for_queue(), **self._set_up_env_for_redis(), @@ -581,7 +578,8 @@ def _set_up_env(self): def get_or_create_instance_id(clp_config: CLPConfig): """ - Gets or create a unique instance ID for this CLP instance. + Gets or creates a unique instance ID for this CLP instance. + :param clp_config: :return: The instance ID. """ From 5bf08873fdb05c2c726c10c606694a92fcbbf32b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 03:17:02 -0400 Subject: [PATCH 144/238] refactor(controller): Rename clp_config -> _clp_config for consistent private attribute naming. --- .../clp_package_utils/controller.py | 148 +++++++++--------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 6162022266..84ecfb3726 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -68,7 +68,7 @@ class BaseController(ABC): """ def __init__(self, clp_config: CLPConfig): - self.clp_config = clp_config + self._clp_config = clp_config self._clp_home = get_clp_home() self._conf_dir = self._clp_home / "etc" @@ -104,9 +104,9 @@ def _set_up_env_for_database(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") conf_logging_file = self._conf_dir / "mysql" / "conf.d" / "logging.cnf" - data_dir = self.clp_config.data_directory / component_name - logs_dir = self.clp_config.logs_directory / component_name - validate_db_config(self.clp_config, conf_logging_file, data_dir, logs_dir) + data_dir = self._clp_config.data_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name + validate_db_config(self._clp_config, conf_logging_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) @@ -116,13 +116,13 @@ def _set_up_env_for_database(self) -> EnvVarsDict: "CLP_DB_CONF_LOGGING_FILE_HOST": str(conf_logging_file), "CLP_DB_DATA_DIR_HOST": str(data_dir), "CLP_DB_LOGS_DIR_HOST": str(logs_dir), - "CLP_DB_HOST": _get_ip_from_hostname(self.clp_config.database.host), - "CLP_DB_PORT": str(self.clp_config.database.port), - "CLP_DB_NAME": self.clp_config.database.name, - "CLP_DB_USER": self.clp_config.database.username, - "CLP_DB_PASS": self.clp_config.database.password, + "CLP_DB_HOST": _get_ip_from_hostname(self._clp_config.database.host), + "CLP_DB_PORT": str(self._clp_config.database.port), + "CLP_DB_NAME": self._clp_config.database.name, + "CLP_DB_USER": self._clp_config.database.username, + "CLP_DB_PASS": self._clp_config.database.password, "CLP_DB_IMAGE": ( - "mysql:8.0.23" if "mysql" == self.clp_config.database.type else "mariadb:10-jammy" + "mysql:8.0.23" if "mysql" == self._clp_config.database.type else "mariadb:10-jammy" ), } @@ -135,18 +135,18 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: component_name = QUEUE_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_dir = self.clp_config.logs_directory / component_name - validate_queue_config(self.clp_config, logs_dir) + logs_dir = self._clp_config.logs_directory / component_name + validate_queue_config(self._clp_config, logs_dir) logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(logs_dir) return { "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), - "CLP_QUEUE_HOST": _get_ip_from_hostname(self.clp_config.queue.host), - "CLP_QUEUE_PORT": str(self.clp_config.queue.port), - "CLP_QUEUE_USER": self.clp_config.queue.username, - "CLP_QUEUE_PASS": self.clp_config.queue.password, + "CLP_QUEUE_HOST": _get_ip_from_hostname(self._clp_config.queue.host), + "CLP_QUEUE_PORT": str(self._clp_config.queue.port), + "CLP_QUEUE_USER": self._clp_config.queue.username, + "CLP_QUEUE_PASS": self._clp_config.queue.password, } def _set_up_env_for_redis(self) -> EnvVarsDict: @@ -159,9 +159,9 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") conf_file = self._conf_dir / "redis" / "redis.conf" - data_dir = self.clp_config.data_directory / component_name - logs_dir = self.clp_config.logs_directory / component_name - validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir = self._clp_config.data_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name + validate_redis_config(self._clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) @@ -171,12 +171,12 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: "CLP_REDIS_CONF_FILE_HOST": str(conf_file), "CLP_REDIS_DATA_DIR_HOST": str(data_dir), "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), - "CLP_REDIS_HOST": _get_ip_from_hostname(self.clp_config.redis.host), - "CLP_REDIS_PORT": str(self.clp_config.redis.port), - "CLP_REDIS_PASS": self.clp_config.redis.password, - "CLP_REDIS_QUERY_BACKEND_DB": str(self.clp_config.redis.query_backend_database), + "CLP_REDIS_HOST": _get_ip_from_hostname(self._clp_config.redis.host), + "CLP_REDIS_PORT": str(self._clp_config.redis.port), + "CLP_REDIS_PASS": self._clp_config.redis.password, + "CLP_REDIS_QUERY_BACKEND_DB": str(self._clp_config.redis.query_backend_database), "CLP_REDIS_COMPRESSION_BACKEND_DB": str( - self.clp_config.redis.compression_backend_database + self._clp_config.redis.compression_backend_database ), } @@ -190,9 +190,9 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") conf_file = self._conf_dir / "mongo" / "mongod.conf" - data_dir = self.clp_config.data_directory / component_name - logs_dir = self.clp_config.logs_directory / component_name - validate_results_cache_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir = self._clp_config.data_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name + validate_results_cache_config(self._clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) @@ -202,10 +202,10 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: "CLP_RESULTS_CACHE_CONF_FILE_HOST": str(conf_file), "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), - "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self.clp_config.results_cache.host), - "CLP_RESULTS_CACHE_PORT": str(self.clp_config.results_cache.port), - "CLP_RESULTS_CACHE_DB_NAME": self.clp_config.results_cache.db_name, - "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.clp_config.results_cache.stream_collection_name, + "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self._clp_config.results_cache.host), + "CLP_RESULTS_CACHE_PORT": str(self._clp_config.results_cache.port), + "CLP_RESULTS_CACHE_DB_NAME": self._clp_config.results_cache.db_name, + "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self._clp_config.results_cache.stream_collection_name, } def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: @@ -217,11 +217,11 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - log_file = self.clp_config.logs_directory / f"{component_name}.log" + log_file = self._clp_config.logs_directory / f"{component_name}.log" log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) return { - "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self.clp_config.compression_scheduler.logging_level, + "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self._clp_config.compression_scheduler.logging_level, "CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST": str(log_file), } @@ -234,11 +234,11 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - log_file = self.clp_config.logs_directory / f"{component_name}.log" + log_file = self._clp_config.logs_directory / f"{component_name}.log" log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) return { - "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self.clp_config.query_scheduler.logging_level, + "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self._clp_config.query_scheduler.logging_level, "CLP_QUERY_SCHEDULER_LOG_FILE_HOST": str(log_file), } @@ -252,13 +252,13 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: component_name = COMPRESSION_WORKER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_dir = self.clp_config.logs_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) return { "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_workers), - "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": self.clp_config.compression_worker.logging_level, + "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": self._clp_config.compression_worker.logging_level, "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), } @@ -272,12 +272,12 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: component_name = QUERY_WORKER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_dir = self.clp_config.logs_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) return { - "CLP_QUERY_WORKER_LOGGING_LEVEL": self.clp_config.query_worker.logging_level, + "CLP_QUERY_WORKER_LOGGING_LEVEL": self._clp_config.query_worker.logging_level, "CLP_QUERY_WORKER_LOGS_DIR_HOST": str(logs_dir), "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } @@ -292,15 +292,15 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: component_name = REDUCER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_dir = self.clp_config.logs_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) return { - "CLP_REDUCER_LOGGING_LEVEL": self.clp_config.reducer.logging_level, + "CLP_REDUCER_LOGGING_LEVEL": self._clp_config.reducer.logging_level, "CLP_REDUCER_LOGS_DIR_HOST": str(logs_dir), "CLP_REDUCER_CONCURRENCY": str(num_workers), - "CLP_REDUCER_UPSERT_INTERVAL": str(self.clp_config.reducer.upsert_interval), + "CLP_REDUCER_UPSERT_INTERVAL": str(self._clp_config.reducer.upsert_interval), } def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: @@ -321,12 +321,12 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" ) - validate_webui_config(self.clp_config, client_settings_json_path, server_settings_json_path) + validate_webui_config(self._clp_config, client_settings_json_path, server_settings_json_path) # Read, update, and write back client's and server's settings.json - clp_db_connection_params = self.clp_config.database.get_clp_connection_params_and_type(True) + clp_db_connection_params = self._clp_config.database.get_clp_connection_params_and_type(True) table_prefix = clp_db_connection_params["table_prefix"] - if StorageEngine.CLP_S == self.clp_config.package.storage_engine: + if StorageEngine.CLP_S == self._clp_config.package.storage_engine: archives_table_name = "" files_table_name = "" else: @@ -334,9 +334,9 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: files_table_name = get_files_table_name(table_prefix, None) client_settings_json_updates = { - "ClpStorageEngine": self.clp_config.package.storage_engine, - "ClpQueryEngine": self.clp_config.package.query_engine, - "MongoDbSearchResultsMetadataCollectionName": self.clp_config.webui.results_metadata_collection_name, + "ClpStorageEngine": self._clp_config.package.storage_engine, + "ClpQueryEngine": self._clp_config.package.query_engine, + "MongoDbSearchResultsMetadataCollectionName": self._clp_config.webui.results_metadata_collection_name, "SqlDbClpArchivesTableName": archives_table_name, "SqlDbClpDatasetsTableName": get_datasets_table_name(table_prefix), "SqlDbClpFilesTableName": files_table_name, @@ -352,19 +352,19 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: server_settings_json_updates = { "SqlDbHost": container_clp_config.database.host, "SqlDbPort": container_clp_config.database.port, - "SqlDbName": self.clp_config.database.name, + "SqlDbName": self._clp_config.database.name, "SqlDbQueryJobsTableName": "query_jobs", "MongoDbHost": container_clp_config.results_cache.host, "MongoDbPort": container_clp_config.results_cache.port, - "MongoDbName": self.clp_config.results_cache.db_name, - "MongoDbSearchResultsMetadataCollectionName": self.clp_config.webui.results_metadata_collection_name, - "MongoDbStreamFilesCollectionName": self.clp_config.results_cache.stream_collection_name, + "MongoDbName": self._clp_config.results_cache.db_name, + "MongoDbSearchResultsMetadataCollectionName": self._clp_config.webui.results_metadata_collection_name, + "MongoDbStreamFilesCollectionName": self._clp_config.results_cache.stream_collection_name, "ClientDir": str(container_webui_dir / "client"), "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), - "StreamTargetUncompressedSize": self.clp_config.stream_output.target_uncompressed_size, + "StreamTargetUncompressedSize": self._clp_config.stream_output.target_uncompressed_size, } - stream_storage = self.clp_config.stream_output.storage + stream_storage = self._clp_config.stream_output.storage if StorageType.S3 == stream_storage.type: s3_config = stream_storage.s3_config server_settings_json_updates["StreamFilesDir"] = None @@ -385,10 +385,10 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: server_settings_json_updates["StreamFilesS3PathPrefix"] = None server_settings_json_updates["StreamFilesS3Profile"] = None - query_engine = self.clp_config.package.query_engine + query_engine = self._clp_config.package.query_engine if QueryEngine.PRESTO == query_engine: - server_settings_json_updates["PrestoHost"] = self.clp_config.presto.host - server_settings_json_updates["PrestoPort"] = self.clp_config.presto.port + server_settings_json_updates["PrestoHost"] = self._clp_config.presto.host + server_settings_json_updates["PrestoPort"] = self._clp_config.presto.port else: server_settings_json_updates["PrestoHost"] = None server_settings_json_updates["PrestoPort"] = None @@ -400,9 +400,9 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: settings_json_file.write(json.dumps(server_settings_json)) return { - "CLP_WEBUI_HOST": _get_ip_from_hostname(self.clp_config.webui.host), - "CLP_WEBUI_PORT": str(self.clp_config.webui.port), - "CLP_WEBUI_RATE_LIMIT": str(self.clp_config.webui.rate_limit), + "CLP_WEBUI_HOST": _get_ip_from_hostname(self._clp_config.webui.host), + "CLP_WEBUI_PORT": str(self._clp_config.webui.port), + "CLP_WEBUI_RATE_LIMIT": str(self._clp_config.webui.rate_limit), } def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: @@ -414,11 +414,11 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: component_name = GARBAGE_COLLECTOR_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_dir = self.clp_config.logs_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return {"CLP_GC_LOGGING_LEVEL": self.clp_config.garbage_collector.logging_level} + return {"CLP_GC_LOGGING_LEVEL": self._clp_config.garbage_collector.logging_level} def _read_and_update_settings_json( self, settings_file_path: pathlib.Path, updates: Dict[str, Any] @@ -477,7 +477,7 @@ def start(self): check_docker_dependencies(should_compose_run=False, project_name=self._project_name) self._set_up_env() - deployment_type = self.clp_config.get_deployment_type() + deployment_type = self._clp_config.get_deployment_type() logger.info(f"Starting CLP using Docker Compose ({deployment_type} deployment)...") cmd = ["docker", "compose", "--project-name", self._project_name] @@ -529,28 +529,28 @@ def _set_up_env(self): - Preparing environment variables for all components. - Writing environment variables to `.env`. """ - container_clp_config = generate_docker_compose_container_config(self.clp_config) + container_clp_config = generate_docker_compose_container_config(self._clp_config) num_workers = self._get_num_workers() - dump_shared_container_config(container_clp_config, self.clp_config) + dump_shared_container_config(container_clp_config, self._clp_config) env_dict = { - "CLP_PACKAGE_STORAGE_ENGINE": self.clp_config.package.storage_engine, + "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, # User and group IDs "CLP_UID_GID": DEFAULT_UID_GID, "CLP_SERVICE_CONTAINER_UID_GID": ( SERVICE_CONTAINER_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID ), # Package container - "CLP_PACKAGE_CONTAINER": self.clp_config.container_image_ref, + "CLP_PACKAGE_CONTAINER": self._clp_config.container_image_ref, # Runtime data directories - "CLP_DATA_DIR_HOST": str(self.clp_config.data_directory), - "CLP_LOGS_DIR_HOST": str(self.clp_config.logs_directory), + "CLP_DATA_DIR_HOST": str(self._clp_config.data_directory), + "CLP_LOGS_DIR_HOST": str(self._clp_config.logs_directory), # Input directories - "CLP_LOGS_INPUT_DIR_HOST": str(self.clp_config.logs_input.directory), + "CLP_LOGS_INPUT_DIR_HOST": str(self._clp_config.logs_input.directory), "CLP_LOGS_INPUT_DIR_CONTAINER": str(container_clp_config.logs_input.directory), # Output directories - "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self.clp_config.archive_output.get_directory()), - "CLP_STREAM_OUTPUT_DIR_HOST": str(self.clp_config.stream_output.get_directory()), + "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self._clp_config.archive_output.get_directory()), + "CLP_STREAM_OUTPUT_DIR_HOST": str(self._clp_config.stream_output.get_directory()), # AWS credentials "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), @@ -568,8 +568,8 @@ def _set_up_env(self): **self._set_up_env_for_garbage_collector(), } - if self.clp_config.aws_config_directory is not None: - env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self.clp_config.aws_config_directory) + if self._clp_config.aws_config_directory is not None: + env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self._clp_config.aws_config_directory) with open(f"{self._clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): From 26dfacacf2856d997745db31b81719414b8b684a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 03:37:25 -0400 Subject: [PATCH 145/238] refactor(deployment): Rename Redis environment variables for improved clarity. --- components/clp-package-utils/clp_package_utils/controller.py | 4 ++-- tools/deployment/package/docker-compose.base.yaml | 4 ++-- tools/deployment/package/docker-compose.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 84ecfb3726..7d5c84f96c 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -174,8 +174,8 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: "CLP_REDIS_HOST": _get_ip_from_hostname(self._clp_config.redis.host), "CLP_REDIS_PORT": str(self._clp_config.redis.port), "CLP_REDIS_PASS": self._clp_config.redis.password, - "CLP_REDIS_QUERY_BACKEND_DB": str(self._clp_config.redis.query_backend_database), - "CLP_REDIS_COMPRESSION_BACKEND_DB": str( + "CLP_REDIS_BACKEND_DB_QUERY": str(self._clp_config.redis.query_backend_database), + "CLP_REDIS_BACKEND_DB_COMPRESSION": str( self._clp_config.redis.compression_backend_database ), } diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 2c00c58f15..03e1dcf867 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -216,7 +216,7 @@ services: CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly @@ -252,7 +252,7 @@ services: CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 44ecae5e89..b66757980a 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -40,7 +40,7 @@ services: CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_clp_config_readonly - type: "bind" @@ -82,7 +82,7 @@ services: CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly From 99e983f29e5f3a79a0efe0c491125c62c1ac2c1a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 03:51:02 -0400 Subject: [PATCH 146/238] Apply suggestions - Reformat multiline statements and remove unnecessary blank lines. --- .../clp_package_utils/controller.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 7d5c84f96c..96f78dbd89 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -205,7 +205,9 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self._clp_config.results_cache.host), "CLP_RESULTS_CACHE_PORT": str(self._clp_config.results_cache.port), "CLP_RESULTS_CACHE_DB_NAME": self._clp_config.results_cache.db_name, - "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self._clp_config.results_cache.stream_collection_name, + "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": ( + self._clp_config.results_cache.stream_collection_name + ), } def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: @@ -253,7 +255,6 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) return { @@ -273,7 +274,6 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) return { @@ -293,7 +293,6 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) return { @@ -320,11 +319,14 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: server_settings_json_path = ( self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" ) - - validate_webui_config(self._clp_config, client_settings_json_path, server_settings_json_path) + validate_webui_config( + self._clp_config, client_settings_json_path, server_settings_json_path + ) # Read, update, and write back client's and server's settings.json - clp_db_connection_params = self._clp_config.database.get_clp_connection_params_and_type(True) + clp_db_connection_params = self._clp_config.database.get_clp_connection_params_and_type( + True + ) table_prefix = clp_db_connection_params["table_prefix"] if StorageEngine.CLP_S == self._clp_config.package.storage_engine: archives_table_name = "" @@ -415,7 +417,6 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) return {"CLP_GC_LOGGING_LEVEL": self._clp_config.garbage_collector.logging_level} From 0b028dcef5f19d58d019a04d53d3904b466c34d6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 03:58:19 -0400 Subject: [PATCH 147/238] Apply suggestions - Rename UID/GID environment variables for improved clarity. --- .../clp_package_utils/controller.py | 14 +++++++------- tools/deployment/package/docker-compose.base.yaml | 10 +++++----- tools/deployment/package/docker-compose.yaml | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 96f78dbd89..0d0b19ca1f 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -53,9 +53,9 @@ LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH DEFAULT_UID_GID = f"{os.getuid()}:{os.getgid()}" -SERVICE_CONTAINER_USER_ID = 999 -SERVICE_CONTAINER_GROUP_ID = 999 -SERVICE_CONTAINER_UID_GID = f"{SERVICE_CONTAINER_USER_ID}:{SERVICE_CONTAINER_GROUP_ID}" +THIRD_PARTY_SERVICE_UID = 999 +THIRD_PARTY_SERVICE_GID = 999 +THIRD_PARTY_SERVICE_UID_GID = f"{THIRD_PARTY_SERVICE_UID}:{THIRD_PARTY_SERVICE_GID}" logger = logging.getLogger(__name__) @@ -537,9 +537,9 @@ def _set_up_env(self): env_dict = { "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, # User and group IDs - "CLP_UID_GID": DEFAULT_UID_GID, - "CLP_SERVICE_CONTAINER_UID_GID": ( - SERVICE_CONTAINER_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID + "CLP_FIRST_PARTY_SERVICE_UID_GID": DEFAULT_UID_GID, + "CLP_THIRD_PARTY_SERVICE_UID_GID": ( + THIRD_PARTY_SERVICE_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID ), # Package container "CLP_PACKAGE_CONTAINER": self._clp_config.container_image_ref, @@ -607,7 +607,7 @@ def _chown_paths_if_root(*paths: pathlib.Path): if os.getuid() != 0: return for path in paths: - _chown_recursively(path, SERVICE_CONTAINER_USER_ID, SERVICE_CONTAINER_GROUP_ID) + _chown_recursively(path, THIRD_PARTY_SERVICE_UID, THIRD_PARTY_SERVICE_GID) def _chown_recursively( diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 03e1dcf867..205aa9c387 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -6,7 +6,7 @@ x-service-defaults: &service_defaults logging: driver: "local" stop_grace_period: "3s" - user: "${CLP_UID_GID:-1000:1000}" + user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:1000}" # Common healthcheck defaults. x-healthcheck-defaults: &healthcheck_defaults @@ -46,7 +46,7 @@ services: <<: *service_defaults image: "${CLP_DB_IMAGE:-mysql:8.0.23}" hostname: "database" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: MYSQL_DATABASE: "${CLP_DB_NAME}" MYSQL_PASSWORD: "${CLP_DB_PASS}" @@ -102,7 +102,7 @@ services: <<: *service_defaults image: "rabbitmq:3.9.8" hostname: "queue" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" @@ -126,7 +126,7 @@ services: <<: *service_defaults image: "redis:7.2.4" hostname: "redis" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" ports: - host_ip: "${CLP_REDIS_HOST:-127.0.0.1}" published: "${CLP_REDIS_PORT:-6379}" @@ -162,7 +162,7 @@ services: <<: *service_defaults image: "mongo:7.0.1" hostname: "results_cache" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" ports: - host_ip: "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}" published: "${CLP_RESULTS_CACHE_PORT:-27017}" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index b66757980a..e803fd3cf7 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -9,7 +9,7 @@ x-service-defaults: &service_defaults logging: driver: "local" stop_grace_period: "3s" - user: "${CLP_UID_GID:-1000:1000}" + user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:1000}" x-healthcheck-defaults: &healthcheck_defaults interval: "30s" retries: 3 From f11963c338eb9ce9ab04c3c99b0bbca17685ced3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 04:03:49 -0400 Subject: [PATCH 148/238] Apply suggestions - Use constants for jobs table names. --- .../clp-package-utils/clp_package_utils/controller.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 0d0b19ca1f..cde2c9d071 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -13,11 +13,13 @@ from clp_py_utils.clp_config import ( AwsAuthType, CLPConfig, + COMPRESSION_JOBS_TABLE_NAME, COMPRESSION_SCHEDULER_COMPONENT_NAME, COMPRESSION_WORKER_COMPONENT_NAME, DB_COMPONENT_NAME, DeploymentType, GARBAGE_COLLECTOR_COMPONENT_NAME, + QUERY_JOBS_TABLE_NAME, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, QueryEngine, @@ -343,7 +345,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: "SqlDbClpDatasetsTableName": get_datasets_table_name(table_prefix), "SqlDbClpFilesTableName": files_table_name, "SqlDbClpTablePrefix": table_prefix, - "SqlDbCompressionJobsTableName": "compression_jobs", + "SqlDbCompressionJobsTableName": COMPRESSION_JOBS_TABLE_NAME, } client_settings_json = self._read_and_update_settings_json( client_settings_json_path, client_settings_json_updates @@ -355,7 +357,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: "SqlDbHost": container_clp_config.database.host, "SqlDbPort": container_clp_config.database.port, "SqlDbName": self._clp_config.database.name, - "SqlDbQueryJobsTableName": "query_jobs", + "SqlDbQueryJobsTableName": QUERY_JOBS_TABLE_NAME, "MongoDbHost": container_clp_config.results_cache.host, "MongoDbPort": container_clp_config.results_cache.port, "MongoDbName": self._clp_config.results_cache.db_name, From 91cb9fc805df0d33c663e6ed2827501a62da9c5d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 15:44:52 -0400 Subject: [PATCH 149/238] refactor(controller): Remove redundant `stderr` redirection from subprocess commands --- components/clp-package-utils/clp_package_utils/controller.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index cde2c9d071..dd977c11bd 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -491,7 +491,6 @@ def start(self): subprocess.run( cmd, cwd=self._clp_home, - stderr=subprocess.STDOUT, check=True, ) except subprocess.CalledProcessError: @@ -509,7 +508,6 @@ def stop(self): subprocess.run( ["docker", "compose", "--project-name", self._project_name, "down"], cwd=self._clp_home, - stderr=subprocess.STDOUT, check=True, ) logger.info("All CLP containers stopped.") From 1d57a8986b6c30e30cd88cbf5ee8a521b8eb3941 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 15:49:51 -0400 Subject: [PATCH 150/238] docs: move high-level comments to individual blocks --- .../clp-package-utils/clp_package_utils/controller.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index dd977c11bd..f549af7084 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -524,16 +524,12 @@ def _get_num_workers() -> int: return multiprocessing.cpu_count() // 2 def _set_up_env(self): - """ - Sets up all CLP components for Docker Compose by: - - Generating container-specific config. - - Preparing environment variables for all components. - - Writing environment variables to `.env`. - """ + # Generate container-specific config. container_clp_config = generate_docker_compose_container_config(self._clp_config) num_workers = self._get_num_workers() dump_shared_container_config(container_clp_config, self._clp_config) + # Prepare environment variables for all components. env_dict = { "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, # User and group IDs @@ -572,6 +568,7 @@ def _set_up_env(self): if self._clp_config.aws_config_directory is not None: env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self._clp_config.aws_config_directory) + # Write the environment variables to the `.env ` file. with open(f"{self._clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): env_file.write(f"{key}={value}\n") From f4254fa78b5518d789ce17752da2a9aeb3b9631f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 15:52:13 -0400 Subject: [PATCH 151/238] docs(controller): reference issue for revisiting worker count logic --- components/clp-package-utils/clp_package_utils/controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index f549af7084..c4cf248d6f 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -518,7 +518,8 @@ def stop(self): @staticmethod def _get_num_workers() -> int: """ - TODO: Revisit after moving from single-container to multi-container workers. + TODO: Revisit after moving from single-container to multi-container workers. See issue + @y-scope/clp#1424 for details. :return: Number of worker processes to run. """ return multiprocessing.cpu_count() // 2 From 932c3343ce3de59b8f547d904f4ffd8f2d091c12 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 16:03:22 -0400 Subject: [PATCH 152/238] refactor(controller): add `--wait` flag to `docker compose up` command to wait for all services become healthy before existing --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index c4cf248d6f..e7bedaef8b 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -486,7 +486,7 @@ def start(self): cmd = ["docker", "compose", "--project-name", self._project_name] if deployment_type == DeploymentType.BASE: cmd += ["--file", "docker-compose.base.yaml"] - cmd += ["up", "--detach"] + cmd += ["up", "--detach", "--wait"] try: subprocess.run( cmd, From a8cee64234aef8bd1b4bf650f827d0073f9e03bb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 20:29:01 -0400 Subject: [PATCH 153/238] refactor(controller): replace `Dict` with a custom `EnvVarsDict` for consistent type-safe environment variable management; group the env vars; replace `Dict` usages with `dict`. --- .../clp_package_utils/controller.py | 266 +++++++++++++----- 1 file changed, 197 insertions(+), 69 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index e7bedaef8b..0a2b1e3fd2 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -8,7 +8,7 @@ import subprocess import uuid from abc import ABC, abstractmethod -from typing import Any, Dict +from typing import Any, Optional from clp_py_utils.clp_config import ( AwsAuthType, @@ -50,8 +50,6 @@ validate_webui_config, ) -EnvVarsDict = Dict[str, str] - LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH DEFAULT_UID_GID = f"{os.getuid()}:{os.getgid()}" @@ -62,6 +60,15 @@ logger = logging.getLogger(__name__) +class EnvVarsDict(dict[str, Optional[str]]): + def __ior__(self, other: "EnvVarsDict") -> "EnvVarsDict": + """ + Overloads the `|=` operator for static type checking on `other`. + """ + super().__ior__(other) + return self + + class BaseController(ABC): """ Base controller for orchestrating CLP components. Derived classes should implement any @@ -114,20 +121,33 @@ def _set_up_env_for_database(self) -> EnvVarsDict: logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) - return { - "CLP_DB_CONF_LOGGING_FILE_HOST": str(conf_logging_file), - "CLP_DB_DATA_DIR_HOST": str(data_dir), - "CLP_DB_LOGS_DIR_HOST": str(logs_dir), + env_vars = EnvVarsDict() + # Connection + env_vars |= { "CLP_DB_HOST": _get_ip_from_hostname(self._clp_config.database.host), - "CLP_DB_PORT": str(self._clp_config.database.port), "CLP_DB_NAME": self._clp_config.database.name, - "CLP_DB_USER": self._clp_config.database.username, + "CLP_DB_PORT": str(self._clp_config.database.port), + } + # Credential + env_vars |= { "CLP_DB_PASS": self._clp_config.database.password, + "CLP_DB_USER": self._clp_config.database.username, + } + # Path + env_vars |= { + "CLP_DB_CONF_LOGGING_FILE_HOST": str(conf_logging_file), + "CLP_DB_DATA_DIR_HOST": str(data_dir), + "CLP_DB_LOGS_DIR_HOST": str(logs_dir), + } + # Runtime + env_vars |= { "CLP_DB_IMAGE": ( - "mysql:8.0.23" if "mysql" == self._clp_config.database.type else "mariadb:10-jammy" + "mysql:8.0.23" if self._clp_config.database.type == "mysql" else "mariadb:10-jammy" ), } + return env_vars + def _set_up_env_for_queue(self) -> EnvVarsDict: """ Sets up environment variables and directories for the message queue component. @@ -143,13 +163,23 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(logs_dir) - return { - "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), + env_vars = EnvVarsDict() + # Connection + env_vars |= { "CLP_QUEUE_HOST": _get_ip_from_hostname(self._clp_config.queue.host), "CLP_QUEUE_PORT": str(self._clp_config.queue.port), + } + # Credential + env_vars |= { "CLP_QUEUE_USER": self._clp_config.queue.username, "CLP_QUEUE_PASS": self._clp_config.queue.password, } + # Path + env_vars |= { + "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), + } + + return env_vars def _set_up_env_for_redis(self) -> EnvVarsDict: """ @@ -169,19 +199,32 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) - return { - "CLP_REDIS_CONF_FILE_HOST": str(conf_file), - "CLP_REDIS_DATA_DIR_HOST": str(data_dir), - "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), - "CLP_REDIS_HOST": _get_ip_from_hostname(self._clp_config.redis.host), - "CLP_REDIS_PORT": str(self._clp_config.redis.port), - "CLP_REDIS_PASS": self._clp_config.redis.password, - "CLP_REDIS_BACKEND_DB_QUERY": str(self._clp_config.redis.query_backend_database), + env_vars = EnvVarsDict() + # Backend + env_vars |= { "CLP_REDIS_BACKEND_DB_COMPRESSION": str( self._clp_config.redis.compression_backend_database ), + "CLP_REDIS_BACKEND_DB_QUERY": str(self._clp_config.redis.query_backend_database), + } + # Connection + env_vars |= { + "CLP_REDIS_HOST": _get_ip_from_hostname(self._clp_config.redis.host), + "CLP_REDIS_PORT": str(self._clp_config.redis.port), + } + # Credential + env_vars |= { + "CLP_REDIS_PASS": self._clp_config.redis.password, + } + # Path + env_vars |= { + "CLP_REDIS_CONF_FILE_HOST": str(conf_file), + "CLP_REDIS_DATA_DIR_HOST": str(data_dir), + "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), } + return env_vars + def _set_up_env_for_results_cache(self) -> EnvVarsDict: """ Sets up environment variables and directories for the results cache (MongoDB) component. @@ -200,17 +243,27 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) - return { - "CLP_RESULTS_CACHE_CONF_FILE_HOST": str(conf_file), - "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), - "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), - "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self._clp_config.results_cache.host), - "CLP_RESULTS_CACHE_PORT": str(self._clp_config.results_cache.port), - "CLP_RESULTS_CACHE_DB_NAME": self._clp_config.results_cache.db_name, + env_vars = EnvVarsDict() + # Collection + env_vars |= { "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": ( self._clp_config.results_cache.stream_collection_name ), } + # Connection + env_vars |= { + "CLP_RESULTS_CACHE_DB_NAME": self._clp_config.results_cache.db_name, + "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self._clp_config.results_cache.host), + "CLP_RESULTS_CACHE_PORT": str(self._clp_config.results_cache.port), + } + # Path + env_vars |= { + "CLP_RESULTS_CACHE_CONF_FILE_HOST": str(conf_file), + "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), + "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), + } + + return env_vars def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: """ @@ -224,11 +277,20 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: log_file = self._clp_config.logs_directory / f"{component_name}.log" log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) - return { - "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self._clp_config.compression_scheduler.logging_level, + env_vars = EnvVarsDict() + # Logging + env_vars |= { + "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": ( + self._clp_config.compression_scheduler.logging_level + ), + } + # Path + env_vars |= { "CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST": str(log_file), } + return env_vars + def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: """ Sets up environment variables and files for the query scheduler component. @@ -241,11 +303,18 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: log_file = self._clp_config.logs_directory / f"{component_name}.log" log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) - return { + env_vars = EnvVarsDict() + # Logging + env_vars |= { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self._clp_config.query_scheduler.logging_level, + } + # Path + env_vars |= { "CLP_QUERY_SCHEDULER_LOG_FILE_HOST": str(log_file), } + return env_vars + def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: """ Sets up environment variables for the compression worker component. @@ -259,11 +328,23 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return { - "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_workers), - "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": self._clp_config.compression_worker.logging_level, + env_vars = EnvVarsDict() + # Logging + env_vars |= { + "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": ( + self._clp_config.compression_worker.logging_level + ), + } + # Path + env_vars |= { "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), } + # Resource + env_vars |= { + "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_workers), + } + + return env_vars def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: """ @@ -278,12 +359,22 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return { + env_vars = EnvVarsDict() + # Logging + env_vars |= { "CLP_QUERY_WORKER_LOGGING_LEVEL": self._clp_config.query_worker.logging_level, + } + # Path + env_vars |= { "CLP_QUERY_WORKER_LOGS_DIR_HOST": str(logs_dir), + } + # Resource + env_vars |= { "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } + return env_vars + def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: """ Sets up environment variables for the reducer component. @@ -297,13 +388,23 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return { + env_vars = EnvVarsDict() + # Logging + env_vars |= { "CLP_REDUCER_LOGGING_LEVEL": self._clp_config.reducer.logging_level, + } + # Path + env_vars |= { "CLP_REDUCER_LOGS_DIR_HOST": str(logs_dir), + } + # Resource + env_vars |= { "CLP_REDUCER_CONCURRENCY": str(num_workers), "CLP_REDUCER_UPSERT_INTERVAL": str(self._clp_config.reducer.upsert_interval), } + return env_vars + def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: """ Sets up environment variables and settings for the Web UI component. @@ -403,12 +504,19 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: with open(server_settings_json_path, "w") as settings_json_file: settings_json_file.write(json.dumps(server_settings_json)) - return { + env_vars = EnvVarsDict() + # Connection + env_vars |= { "CLP_WEBUI_HOST": _get_ip_from_hostname(self._clp_config.webui.host), "CLP_WEBUI_PORT": str(self._clp_config.webui.port), + } + # Security + env_vars |= { "CLP_WEBUI_RATE_LIMIT": str(self._clp_config.webui.rate_limit), } + return env_vars + def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: """ Sets up environment variables for the garbage collector component. @@ -421,11 +529,16 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return {"CLP_GC_LOGGING_LEVEL": self._clp_config.garbage_collector.logging_level} + env_vars = EnvVarsDict() + + # Logging + env_vars |= {"CLP_GC_LOGGING_LEVEL": self._clp_config.garbage_collector.logging_level} + + return env_vars def _read_and_update_settings_json( - self, settings_file_path: pathlib.Path, updates: Dict[str, Any] - ) -> Dict[str, Any]: + self, settings_file_path: pathlib.Path, updates: dict[str, Any] + ) -> dict[str, Any]: """ Reads and updates a settings JSON file. @@ -441,8 +554,8 @@ def _read_and_update_settings_json( def _update_settings_object( self, parent_key_prefix: str, - settings: Dict[str, Any], - updates: Dict[str, Any], + settings: dict[str, Any], + updates: dict[str, Any], ): """ Recursively updates the given settings object with the values from `updates`. @@ -531,47 +644,62 @@ def _set_up_env(self): dump_shared_container_config(container_clp_config, self._clp_config) # Prepare environment variables for all components. - env_dict = { - "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, - # User and group IDs + env_vars = EnvVarsDict() + + # Credential + env_vars |= { + "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), + "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), + } + + # Identity + env_vars |= { "CLP_FIRST_PARTY_SERVICE_UID_GID": DEFAULT_UID_GID, "CLP_THIRD_PARTY_SERVICE_UID_GID": ( THIRD_PARTY_SERVICE_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID ), - # Package container + } + + # Package + env_vars |= { "CLP_PACKAGE_CONTAINER": self._clp_config.container_image_ref, - # Runtime data directories + "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, + } + + # Path + aws_config_dir = self._clp_config.aws_config_directory + env_vars |= { + # General "CLP_DATA_DIR_HOST": str(self._clp_config.data_directory), "CLP_LOGS_DIR_HOST": str(self._clp_config.logs_directory), - # Input directories - "CLP_LOGS_INPUT_DIR_HOST": str(self._clp_config.logs_input.directory), + # Config + "CLP_AWS_CONFIG_DIR_HOST": (None if aws_config_dir is None else str(aws_config_dir)), + # Input "CLP_LOGS_INPUT_DIR_CONTAINER": str(container_clp_config.logs_input.directory), - # Output directories + "CLP_LOGS_INPUT_DIR_HOST": str(self._clp_config.logs_input.directory), + # Output "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self._clp_config.archive_output.get_directory()), "CLP_STREAM_OUTPUT_DIR_HOST": str(self._clp_config.stream_output.get_directory()), - # AWS credentials - "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), - "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - # Component-specific environment variables - **self._set_up_env_for_database(), - **self._set_up_env_for_queue(), - **self._set_up_env_for_redis(), - **self._set_up_env_for_results_cache(), - **self._set_up_env_for_compression_scheduler(), - **self._set_up_env_for_query_scheduler(), - **self._set_up_env_for_compression_worker(num_workers), - **self._set_up_env_for_query_worker(num_workers), - **self._set_up_env_for_reducer(num_workers), - **self._set_up_env_for_webui(container_clp_config), - **self._set_up_env_for_garbage_collector(), - } - - if self._clp_config.aws_config_directory is not None: - env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self._clp_config.aws_config_directory) + } + + # Component-specific + env_vars |= self._set_up_env_for_database() + env_vars |= self._set_up_env_for_queue() + env_vars |= self._set_up_env_for_redis() + env_vars |= self._set_up_env_for_results_cache() + env_vars |= self._set_up_env_for_compression_scheduler() + env_vars |= self._set_up_env_for_query_scheduler() + env_vars |= self._set_up_env_for_compression_worker(num_workers) + env_vars |= self._set_up_env_for_query_worker(num_workers) + env_vars |= self._set_up_env_for_reducer(num_workers) + env_vars |= self._set_up_env_for_webui(container_clp_config) + env_vars |= self._set_up_env_for_garbage_collector() # Write the environment variables to the `.env ` file. with open(f"{self._clp_home}/.env", "w") as env_file: - for key, value in env_dict.items(): + for key, value in env_vars.items(): + if value is None: + continue env_file.write(f"{key}={value}\n") From f0e174040a7bcdd455bd4c82affa3bf7e9c7d8cc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 20:30:22 -0400 Subject: [PATCH 154/238] lint --- .../clp_package_utils/controller.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 0a2b1e3fd2..c31ffc444d 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -441,7 +441,9 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: client_settings_json_updates = { "ClpStorageEngine": self._clp_config.package.storage_engine, "ClpQueryEngine": self._clp_config.package.query_engine, - "MongoDbSearchResultsMetadataCollectionName": self._clp_config.webui.results_metadata_collection_name, + "MongoDbSearchResultsMetadataCollectionName": ( + self._clp_config.webui.results_metadata_collection_name + ), "SqlDbClpArchivesTableName": archives_table_name, "SqlDbClpDatasetsTableName": get_datasets_table_name(table_prefix), "SqlDbClpFilesTableName": files_table_name, @@ -462,8 +464,12 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: "MongoDbHost": container_clp_config.results_cache.host, "MongoDbPort": container_clp_config.results_cache.port, "MongoDbName": self._clp_config.results_cache.db_name, - "MongoDbSearchResultsMetadataCollectionName": self._clp_config.webui.results_metadata_collection_name, - "MongoDbStreamFilesCollectionName": self._clp_config.results_cache.stream_collection_name, + "MongoDbSearchResultsMetadataCollectionName": ( + self._clp_config.webui.results_metadata_collection_name + ), + "MongoDbStreamFilesCollectionName": ( + self._clp_config.results_cache.stream_collection_name + ), "ClientDir": str(container_webui_dir / "client"), "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), "StreamTargetUncompressedSize": self._clp_config.stream_output.target_uncompressed_size, From 6679c2dad4d4bf9299ec70bea0a85c1d420d3f32 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 20:34:06 -0400 Subject: [PATCH 155/238] use local variable in assignment in _update_settings_object() --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index c31ffc444d..3e18b47d46 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -580,7 +580,7 @@ def _update_settings_object( if isinstance(value, dict): self._update_settings_object(f"{parent_key_prefix}{key}.", settings[key], value) else: - settings[key] = updates[key] + settings[key] = value class DockerComposeController(BaseController): From 416c31ae61d805085be26a2ef73f6c09fa7a7cd6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 20:40:01 -0400 Subject: [PATCH 156/238] refactor(controller): remove redundant try-except blocks around `subprocess.run` and update docstrings accordingly --- .../clp_package_utils/controller.py | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 3e18b47d46..566ecd54b1 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -595,6 +595,8 @@ def __init__(self, clp_config: CLPConfig, instance_id: str): def start(self): """ Starts CLP's components using Docker Compose. + + :raise: Propagates `subprocess.run`'s exceptions. """ check_docker_dependencies(should_compose_run=False, project_name=self._project_name) self._set_up_env() @@ -606,33 +608,28 @@ def start(self): if deployment_type == DeploymentType.BASE: cmd += ["--file", "docker-compose.base.yaml"] cmd += ["up", "--detach", "--wait"] - try: - subprocess.run( - cmd, - cwd=self._clp_home, - check=True, - ) - except subprocess.CalledProcessError: - logger.exception("Failed to start CLP.") - raise + subprocess.run( + cmd, + cwd=self._clp_home, + check=True, + ) + logger.info("CLP is started.") def stop(self): """ Stops CLP components deployed via Docker Compose. + + :raise: Propagates `subprocess.run`'s exceptions. """ check_docker_dependencies(should_compose_run=True, project_name=self._project_name) logger.info("Stopping all CLP containers using Docker Compose...") - try: - subprocess.run( - ["docker", "compose", "--project-name", self._project_name, "down"], - cwd=self._clp_home, - check=True, - ) - logger.info("All CLP containers stopped.") - except subprocess.CalledProcessError: - logger.exception("Failed to stop CLP containers using Docker Compose.") - raise + subprocess.run( + ["docker", "compose", "--project-name", self._project_name, "down"], + cwd=self._clp_home, + check=True, + ) + logger.info("All CLP containers stopped.") @staticmethod def _get_num_workers() -> int: From 9ab85d2f05917802f96ff3b126409a3acd63ed37 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 21:00:15 -0400 Subject: [PATCH 157/238] add warning log for failed Docker dependency check before attempting container stop --- .../clp_package_utils/controller.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 566ecd54b1..424b6bc66c 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -621,9 +621,17 @@ def stop(self): :raise: Propagates `subprocess.run`'s exceptions. """ - check_docker_dependencies(should_compose_run=True, project_name=self._project_name) + try: + check_docker_dependencies(should_compose_run=True, project_name=self._project_name) + except EnvironmentError as e: + logger.warning( + 'Docker dependencies check failed: "%s". Attempting to stop CLP containers ' + "anyway...", + e, + ) + else: + logger.info("Stopping all CLP containers using Docker Compose...") - logger.info("Stopping all CLP containers using Docker Compose...") subprocess.run( ["docker", "compose", "--project-name", self._project_name, "down"], cwd=self._clp_home, From 1345de36c807d2849774fbdf0fc1f2468a504d1d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 21:08:44 -0400 Subject: [PATCH 158/238] refactor(controller): add explicit return type annotations for all methods and functions --- .../clp_package_utils/controller.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 424b6bc66c..8e5bedda4a 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -76,27 +76,27 @@ class BaseController(ABC): variables, directories, and configuration files for each component. """ - def __init__(self, clp_config: CLPConfig): + def __init__(self, clp_config: CLPConfig) -> None: self._clp_config = clp_config self._clp_home = get_clp_home() self._conf_dir = self._clp_home / "etc" @abstractmethod - def start(self): + def start(self) -> None: """ Starts the components. """ pass @abstractmethod - def stop(self): + def stop(self) -> None: """ Stops the components. """ pass @abstractmethod - def _set_up_env(self): + def _set_up_env(self) -> None: """ Sets up all components to run by preparing environment variables, directories, and configuration files. @@ -562,7 +562,7 @@ def _update_settings_object( parent_key_prefix: str, settings: dict[str, Any], updates: dict[str, Any], - ): + ) -> None: """ Recursively updates the given settings object with the values from `updates`. @@ -588,11 +588,11 @@ class DockerComposeController(BaseController): Controller for orchestrating CLP components using Docker Compose. """ - def __init__(self, clp_config: CLPConfig, instance_id: str): + def __init__(self, clp_config: CLPConfig, instance_id: str) -> None: self._project_name = f"clp-package-{instance_id}" super().__init__(clp_config) - def start(self): + def start(self) -> None: """ Starts CLP's components using Docker Compose. @@ -615,7 +615,7 @@ def start(self): ) logger.info("CLP is started.") - def stop(self): + def stop(self) -> None: """ Stops CLP components deployed via Docker Compose. @@ -648,7 +648,7 @@ def _get_num_workers() -> int: """ return multiprocessing.cpu_count() // 2 - def _set_up_env(self): + def _set_up_env(self) -> None: # Generate container-specific config. container_clp_config = generate_docker_compose_container_config(self._clp_config) num_workers = self._get_num_workers() @@ -714,7 +714,7 @@ def _set_up_env(self): env_file.write(f"{key}={value}\n") -def get_or_create_instance_id(clp_config: CLPConfig): +def get_or_create_instance_id(clp_config: CLPConfig) -> str: """ Gets or creates a unique instance ID for this CLP instance. @@ -734,7 +734,7 @@ def get_or_create_instance_id(clp_config: CLPConfig): return instance_id -def _chown_paths_if_root(*paths: pathlib.Path): +def _chown_paths_if_root(*paths: pathlib.Path) -> None: """ Changes ownership of the given paths to the default service container user/group IDs if the current process is running as root. @@ -751,7 +751,7 @@ def _chown_recursively( path: pathlib.Path, user_id: int, group_id: int, -): +) -> None: """ Recursively changes the owner of the given path to the given user ID and group ID. From 3c3e23d8365a939406adddae464340e94e019a1d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 21:12:53 -0400 Subject: [PATCH 159/238] refactor(scripts): move `instance_id` assignment inside `try` block in start/stop scripts --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 2 +- .../clp-package-utils/clp_package_utils/scripts/stop_clp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index f84ba7c7a3..c64fb6ca6a 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -74,8 +74,8 @@ def main(argv): logger.exception("Failed to create necessary directories.") return -1 - instance_id = get_or_create_instance_id(clp_config) try: + instance_id = get_or_create_instance_id(clp_config) controller = DockerComposeController(clp_config, instance_id) controller.start() except Exception as ex: diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 5506e81f08..ae4d6c329c 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -22,8 +22,8 @@ def main(): logger.exception("Failed to load config.") return -1 - instance_id = get_or_create_instance_id(clp_config) try: + instance_id = get_or_create_instance_id(clp_config) controller = DockerComposeController(clp_config, instance_id) controller.stop() except: From 7c43aad09d3ea976b9a911ddb40921f7c84fe9a1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 21:21:46 -0400 Subject: [PATCH 160/238] revert `--config` argument removal in `stop_clp` to allow specifying configuration file path --- .../clp_package_utils/scripts/stop_clp.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index ae4d6c329c..105e929df7 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -1,4 +1,6 @@ +import argparse import logging +import pathlib import sys from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH @@ -12,12 +14,23 @@ logger = logging.getLogger(__file__) -def main(): +def main(argv): clp_home = get_clp_home() default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH + args_parser = argparse.ArgumentParser(description="Stops CLP") + args_parser.add_argument( + "--config", + "-c", + default=str(default_config_file_path), + help="CLP package configuration file.", + ) + + parsed_args = args_parser.parse_args(argv[1:]) + try: - clp_config = load_config_file(default_config_file_path, default_config_file_path, clp_home) + config_file_path = pathlib.Path(parsed_args.config) + clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) except: logger.exception("Failed to load config.") return -1 @@ -34,4 +47,4 @@ def main(): if "__main__" == __name__: - sys.exit(main()) + sys.exit(main(sys.argv)) From 63c700ad20d14d40051bf4aeeeedbddcccf2c95b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 21:23:45 -0400 Subject: [PATCH 161/238] refactor(config): Rename CLP_DEFAULT_ARCHIVE_* -> CLP_DEFAULT_ARCHIVES_* and CLP_DEFAULT_STREAM_* -> CLP_DEFAULT_STREAMS_*. --- .../clp-py-utils/clp_py_utils/clp_config.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 41a9ee431d..8615a00039 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -50,10 +50,10 @@ CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH = pathlib.Path("etc") / "clp-config.yml" CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path("etc") / "credentials.yml" CLP_DEFAULT_DATA_DIRECTORY_PATH = pathlib.Path("var") / "data" -CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" -CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" -CLP_DEFAULT_STREAM_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" -CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" +CLP_DEFAULT_ARCHIVES_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" +CLP_DEFAULT_ARCHIVES_STAGING_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" +CLP_DEFAULT_STREAMS_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" +CLP_DEFAULT_STREAMS_STAGING_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" CLP_DEFAULT_LOG_DIRECTORY_PATH = pathlib.Path("var") / "log" CLP_DEFAULT_DATASET_NAME = "default" CLP_METADATA_TABLE_PREFIX = "clp_" @@ -486,31 +486,31 @@ def transform_for_container(self): class ArchiveFsStorage(FsStorage): - directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + directory: pathlib.Path = CLP_DEFAULT_ARCHIVES_DIRECTORY_PATH def transform_for_container(self): - self.directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + self.directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVES_DIRECTORY_PATH class StreamFsStorage(FsStorage): - directory: pathlib.Path = CLP_DEFAULT_STREAM_DIRECTORY_PATH + directory: pathlib.Path = CLP_DEFAULT_STREAMS_DIRECTORY_PATH def transform_for_container(self): - self.directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH + self.directory = pathlib.Path("/") / CLP_DEFAULT_STREAMS_DIRECTORY_PATH class ArchiveS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = CLP_DEFAULT_ARCHIVES_STAGING_DIRECTORY_PATH def transform_for_container(self): - self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVES_STAGING_DIRECTORY_PATH class StreamS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = CLP_DEFAULT_STREAMS_STAGING_DIRECTORY_PATH def transform_for_container(self): - self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_STREAMS_STAGING_DIRECTORY_PATH def _get_directory_from_storage_config( From 0f7c78c015fcd702df6161d2b96d7043250b05d5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 21:47:39 -0400 Subject: [PATCH 162/238] apply docs suggestions --- components/clp-py-utils/clp_py_utils/clp_config.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 8615a00039..c0fc401827 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -804,10 +804,8 @@ def validate_presto_config(self): def transform_for_container(self): """ - Adjusts paths and service hosts for containerized execution. - - Converts all relevant directories to absolute paths inside the container - and updates service hostnames/ports to their container service names. + Converts all relevant directories to absolute paths inside the container and updates + component hostnames and ports to their container service names and default ports. """ self.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH self.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH From 5e0775679713157e4f0d4d6ab199804a9b0627dd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 22:13:32 -0400 Subject: [PATCH 163/238] docs(clp-package-utils): Update docstrings for clarity and consistency. --- .../clp_package_utils/general.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index c428e0b351..71a74441ad 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -135,7 +135,7 @@ def is_docker_compose_running(project_name: str) -> bool: Checks if a Docker Compose project is running. :param project_name: - :return: True if at least one instance is running, else False. + :return: Whether at least one instance is running. :raises EnvironmentError: If Docker Compose is not installed or fails. """ cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] @@ -149,12 +149,13 @@ def is_docker_compose_running(project_name: str) -> bool: def check_docker_dependencies(should_compose_run: bool, project_name: str): """ - Checks if Docker and Docker Compose are installed, and whether Docker Compose is running or not. + Checks if Docker and Docker Compose are installed, and whether a Docker Compose project is + running. :param should_compose_run: :param project_name: The Docker Compose project name to check. - :raises EnvironmentError: If any Docker dependency is not installed or Docker Compose state - does not match expectation. + :raises EnvironmentError: If any Docker dependency is not installed or the Docker Compose + project state doesn't match `should_compose_run`. """ try: subprocess.run( @@ -176,11 +177,11 @@ def check_docker_dependencies(should_compose_run: bool, project_name: str): def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): """ - Validate that a log directory path of a component is valid. + Validates that the logs directory path for a component is valid. :param logs_dir: :param component_name: - :raises ValueError: If the path is invalid or not a directory. + :raises ValueError: If the path is invalid or can't be a directory. """ try: validate_path_could_be_dir(logs_dir) @@ -311,7 +312,7 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig Copies the given config and transforms mount paths and hosts for Docker Compose. :param clp_config: - :return: The container config and the mounts. + :return: The container config. """ container_clp_config = clp_config.model_copy(deep=True) container_clp_config.transform_for_container() @@ -498,7 +499,7 @@ def validate_db_config( ): if not base_config.exists(): raise ValueError( - f"{DB_COMPONENT_NAME} base configuration at {str(base_config)} is missing." + f"{DB_COMPONENT_NAME} configuration file missing: '{base_config}'." ) _validate_data_directory(data_dir, DB_COMPONENT_NAME) _validate_log_directory(logs_dir, DB_COMPONENT_NAME) @@ -541,7 +542,7 @@ def validate_results_cache_config( ): if not base_config.exists(): raise ValueError( - f"{RESULTS_CACHE_COMPONENT_NAME} base configuration at {str(base_config)} is missing." + f"{RESULTS_CACHE_COMPONENT_NAME} configuration file missing: '{base_config}'." ) _validate_data_directory(data_dir, RESULTS_CACHE_COMPONENT_NAME) _validate_log_directory(logs_dir, RESULTS_CACHE_COMPONENT_NAME) From ff4f06c7d2463f77b11c8d1bc7b30186e1929eb2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 22:14:55 -0400 Subject: [PATCH 164/238] remove return type from dump_shared_container_config() --- components/clp-package-utils/clp_package_utils/general.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 71a74441ad..ead1f781e9 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -356,9 +356,7 @@ def dump_container_config( return config_file_path_on_container, config_file_path_on_host -def dump_shared_container_config( - container_clp_config: CLPConfig, clp_config: CLPConfig -) -> Tuple[pathlib.Path, pathlib.Path]: +def dump_shared_container_config(container_clp_config: CLPConfig, clp_config: CLPConfig): """ Dumps the given container config to `CLP_SHARED_CONFIG_FILENAME` in the logs directory, so that it's accessible in the container. @@ -498,9 +496,7 @@ def validate_db_config( clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path ): if not base_config.exists(): - raise ValueError( - f"{DB_COMPONENT_NAME} configuration file missing: '{base_config}'." - ) + raise ValueError(f"{DB_COMPONENT_NAME} configuration file missing: '{base_config}'.") _validate_data_directory(data_dir, DB_COMPONENT_NAME) _validate_log_directory(logs_dir, DB_COMPONENT_NAME) From e381c3d1083882f1fe79fc1ef5ad273810dd397f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 22:16:11 -0400 Subject: [PATCH 165/238] refactor(clp-package-utils): Rename base_config -> component_config in validation functions. --- .../clp_package_utils/general.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index ead1f781e9..a894085189 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -493,10 +493,13 @@ def validate_and_load_redis_credentials_file( def validate_db_config( - clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path + clp_config: CLPConfig, + component_config: pathlib.Path, + data_dir: pathlib.Path, + logs_dir: pathlib.Path, ): - if not base_config.exists(): - raise ValueError(f"{DB_COMPONENT_NAME} configuration file missing: '{base_config}'.") + if not component_config.exists(): + raise ValueError(f"{DB_COMPONENT_NAME} configuration file missing: '{component_config}'.") _validate_data_directory(data_dir, DB_COMPONENT_NAME) _validate_log_directory(logs_dir, DB_COMPONENT_NAME) @@ -510,11 +513,14 @@ def validate_queue_config(clp_config: CLPConfig, logs_dir: pathlib.Path): def validate_redis_config( - clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path + clp_config: CLPConfig, + component_config: pathlib.Path, + data_dir: pathlib.Path, + logs_dir: pathlib.Path, ): - if not base_config.exists(): + if not component_config.exists(): raise ValueError( - f"{REDIS_COMPONENT_NAME} base configuration at {str(base_config)} is missing." + f"{REDIS_COMPONENT_NAME} base configuration at {str(component_config)} is missing." ) _validate_data_directory(data_dir, REDIS_COMPONENT_NAME) _validate_log_directory(logs_dir, REDIS_COMPONENT_NAME) @@ -534,11 +540,14 @@ def validate_reducer_config(clp_config: CLPConfig, logs_dir: pathlib.Path, num_w def validate_results_cache_config( - clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path + clp_config: CLPConfig, + component_config: pathlib.Path, + data_dir: pathlib.Path, + logs_dir: pathlib.Path, ): - if not base_config.exists(): + if not component_config.exists(): raise ValueError( - f"{RESULTS_CACHE_COMPONENT_NAME} configuration file missing: '{base_config}'." + f"{RESULTS_CACHE_COMPONENT_NAME} configuration file missing: '{component_config}'." ) _validate_data_directory(data_dir, RESULTS_CACHE_COMPONENT_NAME) _validate_log_directory(logs_dir, RESULTS_CACHE_COMPONENT_NAME) From f426c82d6866d7c3c513e0f72f80d977929906c0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 22:17:35 -0400 Subject: [PATCH 166/238] Simplify raised error message for missing configuration file: redis --- components/clp-package-utils/clp_package_utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index a894085189..71e3cd3559 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -520,7 +520,7 @@ def validate_redis_config( ): if not component_config.exists(): raise ValueError( - f"{REDIS_COMPONENT_NAME} base configuration at {str(component_config)} is missing." + f"{REDIS_COMPONENT_NAME} configuration file missing: '{component_config}'." ) _validate_data_directory(data_dir, REDIS_COMPONENT_NAME) _validate_log_directory(logs_dir, REDIS_COMPONENT_NAME) From 73ec68f3db4c1b2ea2e7e27f969f49de9943a309 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 22:20:44 -0400 Subject: [PATCH 167/238] Apply suggestion - To alphabetize --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 8e5bedda4a..0a5047e00b 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -171,8 +171,8 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: } # Credential env_vars |= { - "CLP_QUEUE_USER": self._clp_config.queue.username, "CLP_QUEUE_PASS": self._clp_config.queue.password, + "CLP_QUEUE_USER": self._clp_config.queue.username, } # Path env_vars |= { From 4e707ae1a5be399165357cd652a721cdbe435e23 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 23:17:28 -0400 Subject: [PATCH 168/238] Remove individual log file paths and use shared logging volume --- .../clp_package_utils/controller.py | 26 ------------------- .../package/docker-compose.base.yaml | 16 +++++------- tools/deployment/package/docker-compose.yaml | 16 +++++------- 3 files changed, 14 insertions(+), 44 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 0a5047e00b..00ca451e5c 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -274,9 +274,6 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - log_file = self._clp_config.logs_directory / f"{component_name}.log" - log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) - env_vars = EnvVarsDict() # Logging env_vars |= { @@ -284,10 +281,6 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: self._clp_config.compression_scheduler.logging_level ), } - # Path - env_vars |= { - "CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST": str(log_file), - } return env_vars @@ -300,18 +293,11 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - log_file = self._clp_config.logs_directory / f"{component_name}.log" - log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) - env_vars = EnvVarsDict() # Logging env_vars |= { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self._clp_config.query_scheduler.logging_level, } - # Path - env_vars |= { - "CLP_QUERY_SCHEDULER_LOG_FILE_HOST": str(log_file), - } return env_vars @@ -335,10 +321,6 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: self._clp_config.compression_worker.logging_level ), } - # Path - env_vars |= { - "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), - } # Resource env_vars |= { "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_workers), @@ -364,10 +346,6 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: env_vars |= { "CLP_QUERY_WORKER_LOGGING_LEVEL": self._clp_config.query_worker.logging_level, } - # Path - env_vars |= { - "CLP_QUERY_WORKER_LOGS_DIR_HOST": str(logs_dir), - } # Resource env_vars |= { "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), @@ -393,10 +371,6 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: env_vars |= { "CLP_REDUCER_LOGGING_LEVEL": self._clp_config.reducer.logging_level, } - # Path - env_vars |= { - "CLP_REDUCER_LOGS_DIR_HOST": str(logs_dir), - } # Resource env_vars |= { "CLP_REDUCER_CONCURRENCY": str(num_workers), diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 205aa9c387..a13cf77162 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -35,6 +35,10 @@ x-volume-definitions: source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" target: "/etc/clp-config.yml" read_only: true + clp-logs: &volume_clp_logs + type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}" + target: "/var/log" logs-input-readonly: &volume_root_logs_readonly type: "bind" source: "${CLP_LOGS_INPUT_DIR_HOST:-/}" @@ -220,10 +224,8 @@ services: volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly + - *volume_clp_logs - *volume_root_logs_readonly - - type: "bind" - source: "${CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST:-./var/log/compression_scheduler.log}" - target: "/var/log/compression_scheduler.log" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -256,6 +258,7 @@ services: volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly + - *volume_clp_logs - *volume_root_logs_readonly - type: "bind" source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" @@ -263,9 +266,6 @@ services: - type: "bind" source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}" target: "/var/data/staged-archives" - - type: "bind" - source: "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}" - target: "/var/log/compression_worker" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" @@ -340,9 +340,7 @@ services: PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - *volume_clp_config_readonly - - type: "bind" - source: "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector" - target: "/var/log/garbage_collector" + - *volume_clp_logs depends_on: db-table-creator: condition: "service_completed_successfully" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index e803fd3cf7..c8192b30c7 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -27,6 +27,10 @@ x-volume-definitions: source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" target: "/etc/clp-config.yml" read_only: true + clp-logs: &volume_clp_logs + type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}" + target: "/var/log" services: query-scheduler: @@ -43,9 +47,7 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_clp_config_readonly - - type: "bind" - source: "${CLP_QUERY_SCHEDULER_LOG_FILE_HOST:-./var/log/query_scheduler.log}" - target: "/var/log/query_scheduler.log" + - *volume_clp_logs depends_on: db-table-creator: condition: "service_completed_successfully" @@ -86,12 +88,10 @@ services: volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly + - *volume_clp_logs - type: "bind" source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" target: "/var/data/archives" - - type: "bind" - source: "${CLP_QUERY_WORKER_LOGS_DIR_HOST:-./var/log/query_worker}" - target: "/var/log/query_worker" - type: "bind" source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}" target: "/var/data/staged-streams" @@ -121,9 +121,7 @@ services: PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - *volume_clp_config_readonly - - type: "bind" - source: "${CLP_REDUCER_LOGS_DIR_HOST:-./var/log/reducer}" - target: "/var/log/reducer" + - *volume_clp_logs depends_on: query-scheduler: condition: "service_healthy" From 6f53877fa68bc456d0ff7f57b714dc60d73c50cc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 23:22:26 -0400 Subject: [PATCH 169/238] fix(controller): revert removal of "ClpQueryEngine" in server_settings_json_updates --- components/clp-package-utils/clp_package_utils/controller.py | 1 + 1 file changed, 1 insertion(+) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 00ca451e5c..cdb048f31b 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -447,6 +447,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: "ClientDir": str(container_webui_dir / "client"), "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), "StreamTargetUncompressedSize": self._clp_config.stream_output.target_uncompressed_size, + "ClpQueryEngine": self._clp_config.package.query_engine, } stream_storage = self._clp_config.stream_output.storage From e7addd3114359226d6fd1ec0b1a3c54273c3d666 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 23:25:15 -0400 Subject: [PATCH 170/238] refactor(clp-package-utils): Rename CLP_PACKAGE_CONTAINER -> CLP_PACKAGE_CONTAINER_IMAGE_REF. --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- tools/deployment/package/docker-compose.base.yaml | 2 +- tools/deployment/package/docker-compose.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index cdb048f31b..af3ae11e13 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -648,7 +648,7 @@ def _set_up_env(self) -> None: # Package env_vars |= { - "CLP_PACKAGE_CONTAINER": self._clp_config.container_image_ref, + "CLP_PACKAGE_CONTAINER_IMAGE_REF": self._clp_config.container_image_ref, "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, } diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index a13cf77162..e81deadbaf 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -2,7 +2,7 @@ name: "clp-package-base" # Common service defaults. x-service-defaults: &service_defaults - image: "${CLP_PACKAGE_CONTAINER:-clp-package}" + image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" logging: driver: "local" stop_grace_period: "3s" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index c8192b30c7..e9797f3b24 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -5,7 +5,7 @@ include: ["docker-compose.base.yaml"] # Below x-* definitions are duplicated from docker-compose.base.yaml. Refer to that file for # documentation. x-service-defaults: &service_defaults - image: "${CLP_PACKAGE_CONTAINER:-clp-package}" + image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" logging: driver: "local" stop_grace_period: "3s" From d4e81f327d300d2461a8831d61eb5bd1d3bdbf32 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 00:30:14 -0400 Subject: [PATCH 171/238] fix(docker-compose): use `:?error` syntax to mark required non-empty env vars; add fallback for CLP_DB_NAME; add empty fallbacks for CLP_AWS_ACCESS_KEY_ID and CLP_AWS_SECRET_ACCESS_KEY --- .../package/docker-compose.base.yaml | 48 +++++++++---------- tools/deployment/package/docker-compose.yaml | 16 +++---- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index e81deadbaf..e3a3cd4cef 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -52,10 +52,10 @@ services: hostname: "database" user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: - MYSQL_DATABASE: "${CLP_DB_NAME}" - MYSQL_PASSWORD: "${CLP_DB_PASS}" - MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS}" - MYSQL_USER: "${CLP_DB_USER}" + MYSQL_DATABASE: "${CLP_DB_NAME:-clp-db}" + MYSQL_PASSWORD: "${CLP_DB_PASS:?error}" + MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS:?error}" + MYSQL_USER: "${CLP_DB_USER:?error}" ports: - host_ip: "${CLP_DB_HOST:-127.0.0.1}" published: "${CLP_DB_PORT:-3306}" @@ -78,16 +78,16 @@ services: "mysqladmin", "ping", "--silent", "-h", "127.0.0.1", - "-u", "${CLP_DB_USER}", - "--password=${CLP_DB_PASS}" + "-u", "${CLP_DB_USER:?error}", + "--password=${CLP_DB_PASS:?error}" ] db-table-creator: <<: *service_defaults hostname: "db_table_creator" environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS:?error}" + CLP_DB_USER: "${CLP_DB_USER:?error}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - *volume_clp_config_readonly @@ -108,8 +108,8 @@ services: hostname: "queue" user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: - RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" - RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" + RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS:?error}" + RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER:?error}" RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" ports: - host_ip: "${CLP_QUEUE_HOST:-127.0.0.1}" @@ -153,13 +153,13 @@ services: "redis-cli", "-h", "127.0.0.1", "-p", "6379", - "-a", "${CLP_REDIS_PASS}", + "-a", "${CLP_REDIS_PASS:?error}", "PING" ] command: [ "redis-server", "/usr/local/etc/redis/redis.conf", - "--requirepass", "${CLP_REDIS_PASS}" + "--requirepass", "${CLP_REDIS_PASS:?error}" ] results-cache: @@ -213,14 +213,14 @@ services: <<: *service_defaults hostname: "compression_scheduler" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS:?error}" + CLP_DB_USER: "${CLP_DB_USER:?error}" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} + redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly @@ -244,9 +244,9 @@ services: <<: *service_defaults hostname: "compression_worker" environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" @@ -254,7 +254,7 @@ services: CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} + redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly @@ -286,8 +286,8 @@ services: <<: *service_defaults hostname: "webui" environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS:?error}" + CLP_DB_USER: "${CLP_DB_USER:?error}" HOST: "0.0.0.0" NODE_ENV: "production" NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" @@ -332,8 +332,8 @@ services: <<: *service_defaults hostname: "garbage_collector" environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS:?error}" + CLP_DB_USER: "${CLP_DB_USER:?error}" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/garbage_collector" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index e9797f3b24..d849449c3d 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -37,14 +37,14 @@ services: <<: *service_defaults hostname: "query_scheduler" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS:?error}" + CLP_DB_USER: "${CLP_DB_USER:?error}" CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_clp_config_readonly - *volume_clp_logs @@ -74,9 +74,9 @@ services: <<: *service_defaults hostname: "query_worker" environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" @@ -84,7 +84,7 @@ services: CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly From a5a497dc2a35cd0847af3c9fb6e2368ba725b2a2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 00:32:14 -0400 Subject: [PATCH 172/238] disallow empty `username` and `password` in `Database` because our db init script does not support such; it's a bad practice anyway --- components/clp-py-utils/clp_py_utils/clp_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index c0fc401827..8bb05351fa 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -156,8 +156,8 @@ class Database(BaseModel): auto_commit: bool = False compress: bool = True - username: Optional[str] = None - password: Optional[str] = None + username: Optional[NonEmptyStr] = None + password: Optional[NonEmptyStr] = None def ensure_credentials_loaded(self): if self.username is None or self.password is None: From 0dd11ce872c4b5aec7043443222c652a7abd7247 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 00:33:27 -0400 Subject: [PATCH 173/238] fix(docker-compose): Update default database image to mariadb:10-jammy. --- tools/deployment/package/docker-compose.base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index e3a3cd4cef..92736468c2 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -48,7 +48,7 @@ x-volume-definitions: services: database: <<: *service_defaults - image: "${CLP_DB_IMAGE:-mysql:8.0.23}" + image: "${CLP_DB_IMAGE:-mariadb:10-jammy}" hostname: "database" user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: From d27a0b12450f947e96fe351f7ccf5940648afaa0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 00:58:46 -0400 Subject: [PATCH 174/238] fix(docker-compose): Add stop_grace_period for compression-worker and compression-scheduler. --- tools/deployment/package/docker-compose.base.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 92736468c2..b2e2499fa6 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -212,6 +212,7 @@ services: compression-scheduler: <<: *service_defaults hostname: "compression_scheduler" + stop_grace_period: "300s" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS:?error}" @@ -243,6 +244,7 @@ services: compression-worker: <<: *service_defaults hostname: "compression_worker" + stop_grace_period: "60s" environment: AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" From a6478e6bfed0ac49bb3d05cc5e61a45a6d7e7306 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 01:02:38 -0400 Subject: [PATCH 175/238] apply docs suggestions --- tools/deployment/package/docker-compose.base.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index b2e2499fa6..813302760f 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -12,14 +12,19 @@ x-service-defaults: &service_defaults x-healthcheck-defaults: &healthcheck_defaults # Avoid lowering to prevent excessive resource usage. interval: "30s" + # Mark unhealthy after 3 failed probes. # - In steady state, ( + ) × 3 = ~90s before the service is marked unhealthy. # - From startup, (60s) + ~90s = ~150s before the service is marked unhealthy. retries: 3 + # Frequent checks during startup allow fast transition to healthy. start_interval: "2s" - # Ignore failures for ~15 frequent checks before counting retries. + + # Ignore failures for / ( + ) = ~15 frequent checks before + # counting retries. start_period: "60s" + # Short timeout since no remote communication is expected. timeout: "2s" From 7bbd134d42c0660032a7a87c0285d026949a0287 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 01:03:41 -0400 Subject: [PATCH 176/238] refactor(docker-compose): Rename volume_root_logs_readonly -> volume_logs_input_readonly. --- tools/deployment/package/docker-compose.base.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 813302760f..e9209c8e83 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -44,7 +44,7 @@ x-volume-definitions: type: "bind" source: "${CLP_LOGS_DIR_HOST:-./var/log}" target: "/var/log" - logs-input-readonly: &volume_root_logs_readonly + logs-input-readonly: &volume_logs_input_readonly type: "bind" source: "${CLP_LOGS_INPUT_DIR_HOST:-/}" target: "${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" @@ -231,7 +231,7 @@ services: - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - - *volume_root_logs_readonly + - *volume_logs_input_readonly depends_on: db-table-creator: condition: "service_completed_successfully" @@ -266,7 +266,7 @@ services: - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - - *volume_root_logs_readonly + - *volume_logs_input_readonly - type: "bind" source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" target: "/var/data/archives" From 04e56f0eb7ec18bc38a8a27039d4762841b96ecd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 01:18:56 -0400 Subject: [PATCH 177/238] ensure `healthcheck` goes after `command` --- .../package/docker-compose.base.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index e9209c8e83..9184cb1759 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -151,6 +151,11 @@ services: - type: "bind" source: "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}" target: "/var/log/redis" + command: [ + "redis-server", + "/usr/local/etc/redis/redis.conf", + "--requirepass", "${CLP_REDIS_PASS:?error}" + ] healthcheck: <<: *healthcheck_defaults test: [ @@ -161,11 +166,6 @@ services: "-a", "${CLP_REDIS_PASS:?error}", "PING" ] - command: [ - "redis-server", - "/usr/local/etc/redis/redis.conf", - "--requirepass", "${CLP_REDIS_PASS:?error}" - ] results-cache: <<: *service_defaults @@ -187,16 +187,16 @@ services: - type: "bind" source: "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}" target: "/var/log/mongodb" + command: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0", + ] healthcheck: <<: *healthcheck_defaults test: [ "CMD-SHELL", "echo 'db.runCommand(\"ping\").ok' | mongosh 127.0.0.1:27017/test --quiet" ] - command: [ - "--config", "/etc/mongo/mongod.conf", - "--bind_ip", "0.0.0.0", - ] results-cache-indices-creator: <<: *service_defaults From 0fc91d74e3b20a149e60cc4011111a430568abe1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 01:28:01 -0400 Subject: [PATCH 178/238] fix(docker-compose): Add AWS credentials environment variables in webui service. --- tools/deployment/package/docker-compose.base.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 9184cb1759..b4a7f0ec74 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -293,6 +293,8 @@ services: <<: *service_defaults hostname: "webui" environment: + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" CLP_DB_PASS: "${CLP_DB_PASS:?error}" CLP_DB_USER: "${CLP_DB_USER:?error}" HOST: "0.0.0.0" From f307d942484cbde72a958125eb5753877179782a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 08:55:42 -0400 Subject: [PATCH 179/238] fix(docker-compose): Update environment variables for archive and stream output directories based on storage type; conditionally mount output dirs. --- .../clp_package_utils/controller.py | 21 +++++++++++++------ .../package/docker-compose.base.yaml | 20 ++++++++++-------- tools/deployment/package/docker-compose.yaml | 12 +++-------- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index af3ae11e13..121241984b 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -660,13 +660,22 @@ def _set_up_env(self) -> None: "CLP_LOGS_DIR_HOST": str(self._clp_config.logs_directory), # Config "CLP_AWS_CONFIG_DIR_HOST": (None if aws_config_dir is None else str(aws_config_dir)), - # Input - "CLP_LOGS_INPUT_DIR_CONTAINER": str(container_clp_config.logs_input.directory), - "CLP_LOGS_INPUT_DIR_HOST": str(self._clp_config.logs_input.directory), - # Output - "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self._clp_config.archive_output.get_directory()), - "CLP_STREAM_OUTPUT_DIR_HOST": str(self._clp_config.stream_output.get_directory()), } + # Input + if self._clp_config.logs_input.type == StorageType.FS: + env_vars["CLP_LOGS_INPUT_DIR_CONTAINER"] = str(container_clp_config.logs_input.directory) + env_vars["CLP_LOGS_INPUT_DIR_HOST"] = str(self._clp_config.logs_input.directory) + # Output + archive_output_dir_str = str(self._clp_config.archive_output.get_directory()) + stream_output_dir_str = str(self._clp_config.stream_output.get_directory()) + if self._clp_config.archive_output.storage.type == StorageType.FS: + env_vars["CLP_ARCHIVE_OUTPUT_DIR_HOST"] = archive_output_dir_str + if self._clp_config.archive_output.storage.type == StorageType.S3: + env_vars["CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST"] = archive_output_dir_str + if self._clp_config.stream_output.storage.type == StorageType.FS: + env_vars["CLP_STREAM_OUTPUT_DIR_HOST"] = stream_output_dir_str + if self._clp_config.stream_output.storage.type == StorageType.S3: + env_vars["CLP_STAGED_STREAM_OUTPUT_DIR_HOST"] = stream_output_dir_str # Component-specific env_vars |= self._set_up_env_for_database() diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index b4a7f0ec74..c7f90206f4 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -50,6 +50,14 @@ x-volume-definitions: target: "${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" read_only: true +volumes: + # Dummy volume to use when a bind mount is not desired. + empty: + driver_opts: + device: "tmpfs" + type: "tmpfs" + size: 0 + services: database: <<: *service_defaults @@ -267,12 +275,8 @@ services: - *volume_clp_config_readonly - *volume_clp_logs - *volume_logs_input_readonly - - type: "bind" - source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" - target: "/var/data/archives" - - type: "bind" - source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}" - target: "/var/data/staged-archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:${CLP_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/archives" + - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" @@ -308,9 +312,7 @@ services: target: 4000 volumes: - *volume_aws_config_readonly - - type: "bind" - source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}" - target: "/var/data/streams" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" - type: "bind" source: "./var/www/webui/client/settings.json" target: "/opt/clp/var/www/webui/client/settings.json" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index d849449c3d..9bb71a137a 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -89,15 +89,9 @@ services: - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - - type: "bind" - source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" - target: "/var/data/archives" - - type: "bind" - source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}" - target: "/var/data/staged-streams" - - type: "bind" - source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}" - target: "/var/data/streams" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" + - "${CLP_STAGED_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/staged-streams" command: [ "python3", "-u", From e5a90dda029937daf82ea7f36f9dd5c7a13471e2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 09:08:05 -0400 Subject: [PATCH 180/238] Replace "docker-compose" (usually refer to v1) with "Docker Compose" in error prompts --- components/clp-package-utils/clp_package_utils/general.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 71e3cd3559..7a682156a7 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -144,7 +144,7 @@ def is_docker_compose_running(project_name: str) -> bool: running_instances = json.loads(output) return len(running_instances) >= 1 except subprocess.CalledProcessError: - raise EnvironmentError("docker-compose is not installed or not functioning properly.") + raise EnvironmentError("Docker Compose is not installed or not functioning properly.") def check_docker_dependencies(should_compose_run: bool, project_name: str): @@ -170,9 +170,9 @@ def check_docker_dependencies(should_compose_run: bool, project_name: str): is_running = is_docker_compose_running(project_name) if should_compose_run and not is_running: - raise EnvironmentError("docker-compose is not running.") + raise EnvironmentError(f"Docker Compose project '{project_name}' is not running.") if not should_compose_run and is_running: - raise EnvironmentError("docker-compose is already running.") + raise EnvironmentError("Docker Compose project '{project_name}' is already running.") def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): From 3015ec4a12bdc8590eecf5b69543c991a5449fd4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 09:12:21 -0400 Subject: [PATCH 181/238] lint --- components/clp-package-utils/clp_package_utils/controller.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 121241984b..e01db0ad30 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -663,7 +663,9 @@ def _set_up_env(self) -> None: } # Input if self._clp_config.logs_input.type == StorageType.FS: - env_vars["CLP_LOGS_INPUT_DIR_CONTAINER"] = str(container_clp_config.logs_input.directory) + env_vars["CLP_LOGS_INPUT_DIR_CONTAINER"] = str( + container_clp_config.logs_input.directory + ) env_vars["CLP_LOGS_INPUT_DIR_HOST"] = str(self._clp_config.logs_input.directory) # Output archive_output_dir_str = str(self._clp_config.archive_output.get_directory()) From ff43c78562e2710eb9995b647bc0ece742ee2cad Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 09:13:10 -0400 Subject: [PATCH 182/238] refactor(clp-package-utils): Reorder functions and rename is_docker_compose_running -> _is_docker_compose_running. --- .../clp_package_utils/general.py | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 7a682156a7..e5c0a6d223 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -96,13 +96,6 @@ def __init__(self, clp_home: pathlib.Path, docker_clp_home: pathlib.Path): self.generated_config_file: Optional[DockerMount] = None -def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: - try: - validate_path_could_be_dir(data_dir) - except ValueError as ex: - raise ValueError(f"{component_name} data directory is invalid: {ex}") - - def get_clp_home(): # Determine CLP_HOME from an environment variable or this script's path clp_home = None @@ -130,23 +123,6 @@ def generate_container_name(job_type: str) -> str: return f"clp-{job_type}-{str(uuid.uuid4())[-4:]}" -def is_docker_compose_running(project_name: str) -> bool: - """ - Checks if a Docker Compose project is running. - - :param project_name: - :return: Whether at least one instance is running. - :raises EnvironmentError: If Docker Compose is not installed or fails. - """ - cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - running_instances = json.loads(output) - return len(running_instances) >= 1 - except subprocess.CalledProcessError: - raise EnvironmentError("Docker Compose is not installed or not functioning properly.") - - def check_docker_dependencies(should_compose_run: bool, project_name: str): """ Checks if Docker and Docker Compose are installed, and whether a Docker Compose project is @@ -168,13 +144,37 @@ def check_docker_dependencies(should_compose_run: bool, project_name: str): except subprocess.CalledProcessError: raise EnvironmentError("docker is not installed or available on the path") - is_running = is_docker_compose_running(project_name) + is_running = _is_docker_compose_running(project_name) if should_compose_run and not is_running: raise EnvironmentError(f"Docker Compose project '{project_name}' is not running.") if not should_compose_run and is_running: raise EnvironmentError("Docker Compose project '{project_name}' is already running.") +def _is_docker_compose_running(project_name: str) -> bool: + """ + Checks if a Docker Compose project is running. + + :param project_name: + :return: Whether at least one instance is running. + :raises EnvironmentError: If Docker Compose is not installed or fails. + """ + cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + running_instances = json.loads(output) + return len(running_instances) >= 1 + except subprocess.CalledProcessError: + raise EnvironmentError("Docker Compose is not installed or not functioning properly.") + + +def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: + try: + validate_path_could_be_dir(data_dir) + except ValueError as ex: + raise ValueError(f"{component_name} data directory is invalid: {ex}") + + def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): """ Validates that the logs directory path for a component is valid. From 699979acc3e2e4586fb0a6f2660fc8643f4c5f4e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 09:28:06 -0400 Subject: [PATCH 183/238] fix(docker-compose): Add mounts for archive and stream output in garbage-collector --- tools/deployment/package/docker-compose.base.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index c7f90206f4..01edcae002 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -352,6 +352,8 @@ services: volumes: - *volume_clp_config_readonly - *volume_clp_logs + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" depends_on: db-table-creator: condition: "service_completed_successfully" From 092a86c66dbd4143a509bf9dd14479ffa5016dc3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 09:29:32 -0400 Subject: [PATCH 184/238] refactor(clp-package-utils): Rename _is_docker_compose_running -> _is_docker_compose_project_running. --- components/clp-package-utils/clp_package_utils/general.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index e5c0a6d223..863339a42b 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -144,14 +144,14 @@ def check_docker_dependencies(should_compose_run: bool, project_name: str): except subprocess.CalledProcessError: raise EnvironmentError("docker is not installed or available on the path") - is_running = _is_docker_compose_running(project_name) + is_running = _is_docker_compose_project_running(project_name) if should_compose_run and not is_running: raise EnvironmentError(f"Docker Compose project '{project_name}' is not running.") if not should_compose_run and is_running: raise EnvironmentError("Docker Compose project '{project_name}' is already running.") -def _is_docker_compose_running(project_name: str) -> bool: +def _is_docker_compose_project_running(project_name: str) -> bool: """ Checks if a Docker Compose project is running. From 8d7517174eea6975ca35a851b306cf4a9a41cd13 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 09:47:35 -0400 Subject: [PATCH 185/238] fix(docker-compose): Rename CLP_AWS_ACCESS_KEY_ID -> CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID, CLP_AWS_SECRET_ACCESS_KEY -> CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY; Update AWS environment variable handling based on storage type. --- .../clp-package-utils/clp_package_utils/controller.py | 11 +++++++---- tools/deployment/package/docker-compose.base.yaml | 6 ++---- tools/deployment/package/docker-compose.yaml | 2 -- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index e01db0ad30..d6f21c4468 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -633,10 +633,13 @@ def _set_up_env(self) -> None: env_vars = EnvVarsDict() # Credential - env_vars |= { - "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), - "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - } + if self._clp_config.stream_output.storage.type == StorageType.S3: + stream_output_aws_auth = self._clp_config.stream_output.storage.s3_config.aws_authentication + if stream_output_aws_auth.type == AwsAuthType.credentials: + env_vars |= { + "CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID": stream_output_aws_auth.credentials.access_key_id, + "CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY": stream_output_aws_auth.credentials.secret_access_key, + } # Identity env_vars |= { diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 01edcae002..82cd4e37d4 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -259,8 +259,6 @@ services: hostname: "compression_worker" stop_grace_period: "60s" environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" @@ -297,8 +295,8 @@ services: <<: *service_defaults hostname: "webui" environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" + AWS_ACCESS_KEY_ID: "${CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID:-}" + AWS_SECRET_ACCESS_KEY: "${CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY:-}" CLP_DB_PASS: "${CLP_DB_PASS:?error}" CLP_DB_USER: "${CLP_DB_USER:?error}" HOST: "0.0.0.0" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 9bb71a137a..e227f3ab3b 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -74,8 +74,6 @@ services: <<: *service_defaults hostname: "query_worker" environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" From f53339f63f713e06f771accbd32db5c615edf861 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 10:16:19 -0400 Subject: [PATCH 186/238] fix(docker-compose): Break long volume mount lines for improved readability. --- tools/deployment/package/docker-compose.base.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 82cd4e37d4..0663fba97b 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -273,8 +273,10 @@ services: - *volume_clp_config_readonly - *volume_clp_logs - *volume_logs_input_readonly - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:${CLP_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/archives" - - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ +${CLP_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/archives" + - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ +${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" From 18d11091b3b89ba5f9d22ad6dfae9260ce399c49 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 10:19:35 -0400 Subject: [PATCH 187/238] lint --- .../clp_package_utils/controller.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index d6f21c4468..83f2868e58 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -634,11 +634,17 @@ def _set_up_env(self) -> None: # Credential if self._clp_config.stream_output.storage.type == StorageType.S3: - stream_output_aws_auth = self._clp_config.stream_output.storage.s3_config.aws_authentication + stream_output_aws_auth = ( + self._clp_config.stream_output.storage.s3_config.aws_authentication + ) if stream_output_aws_auth.type == AwsAuthType.credentials: env_vars |= { - "CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID": stream_output_aws_auth.credentials.access_key_id, - "CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY": stream_output_aws_auth.credentials.secret_access_key, + "CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID": ( + stream_output_aws_auth.credentials.access_key_id + ), + "CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY": ( + stream_output_aws_auth.credentials.secret_access_key + ), } # Identity From 9d9bd46d824599c527f1fd1928e85739469814e6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 17 Oct 2025 21:07:11 -0400 Subject: [PATCH 188/238] refactor(clp-package-utils): Rename should_compose_run -> should_compose_project_be_running in check_docker_dependencies(). --- .../clp-package-utils/clp_package_utils/controller.py | 8 ++++++-- components/clp-package-utils/clp_package_utils/general.py | 8 ++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 83f2868e58..f3ed95c172 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -573,7 +573,9 @@ def start(self) -> None: :raise: Propagates `subprocess.run`'s exceptions. """ - check_docker_dependencies(should_compose_run=False, project_name=self._project_name) + check_docker_dependencies( + should_compose_project_be_running=False, project_name=self._project_name + ) self._set_up_env() deployment_type = self._clp_config.get_deployment_type() @@ -597,7 +599,9 @@ def stop(self) -> None: :raise: Propagates `subprocess.run`'s exceptions. """ try: - check_docker_dependencies(should_compose_run=True, project_name=self._project_name) + check_docker_dependencies( + should_compose_project_be_running=True, project_name=self._project_name + ) except EnvironmentError as e: logger.warning( 'Docker dependencies check failed: "%s". Attempting to stop CLP containers ' diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 863339a42b..3ace0c6123 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -123,12 +123,12 @@ def generate_container_name(job_type: str) -> str: return f"clp-{job_type}-{str(uuid.uuid4())[-4:]}" -def check_docker_dependencies(should_compose_run: bool, project_name: str): +def check_docker_dependencies(should_compose_project_be_running: bool, project_name: str): """ Checks if Docker and Docker Compose are installed, and whether a Docker Compose project is running. - :param should_compose_run: + :param should_compose_project_be_running: :param project_name: The Docker Compose project name to check. :raises EnvironmentError: If any Docker dependency is not installed or the Docker Compose project state doesn't match `should_compose_run`. @@ -145,9 +145,9 @@ def check_docker_dependencies(should_compose_run: bool, project_name: str): raise EnvironmentError("docker is not installed or available on the path") is_running = _is_docker_compose_project_running(project_name) - if should_compose_run and not is_running: + if should_compose_project_be_running and not is_running: raise EnvironmentError(f"Docker Compose project '{project_name}' is not running.") - if not should_compose_run and is_running: + if not should_compose_project_be_running and is_running: raise EnvironmentError("Docker Compose project '{project_name}' is already running.") From 127aacc68f3670956aa923eb5566557db47d2bfc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 17 Oct 2025 22:49:19 -0400 Subject: [PATCH 189/238] refactor(clp-package-utils): Reorder private functions --- .../clp_package_utils/general.py | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 3ace0c6123..cb8ef8a367 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -151,44 +151,6 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n raise EnvironmentError("Docker Compose project '{project_name}' is already running.") -def _is_docker_compose_project_running(project_name: str) -> bool: - """ - Checks if a Docker Compose project is running. - - :param project_name: - :return: Whether at least one instance is running. - :raises EnvironmentError: If Docker Compose is not installed or fails. - """ - cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - running_instances = json.loads(output) - return len(running_instances) >= 1 - except subprocess.CalledProcessError: - raise EnvironmentError("Docker Compose is not installed or not functioning properly.") - - -def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: - try: - validate_path_could_be_dir(data_dir) - except ValueError as ex: - raise ValueError(f"{component_name} data directory is invalid: {ex}") - - -def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): - """ - Validates that the logs directory path for a component is valid. - - :param logs_dir: - :param component_name: - :raises ValueError: If the path is invalid or can't be a directory. - """ - try: - validate_path_could_be_dir(logs_dir) - except ValueError as ex: - raise ValueError(f"{component_name} logs directory is invalid: {ex}") - - def validate_port(port_name: str, hostname: str, port: int): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -727,3 +689,41 @@ def get_celery_connection_env_vars_list(container_clp_config: CLPConfig) -> List ] return env_vars + + +def _is_docker_compose_project_running(project_name: str) -> bool: + """ + Checks if a Docker Compose project is running. + + :param project_name: + :return: Whether at least one instance is running. + :raises EnvironmentError: If Docker Compose is not installed or fails. + """ + cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + running_instances = json.loads(output) + return len(running_instances) >= 1 + except subprocess.CalledProcessError: + raise EnvironmentError("Docker Compose is not installed or not functioning properly.") + + +def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: + try: + validate_path_could_be_dir(data_dir) + except ValueError as ex: + raise ValueError(f"{component_name} data directory is invalid: {ex}") + + +def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): + """ + Validates that the logs directory path for a component is valid. + + :param logs_dir: + :param component_name: + :raises ValueError: If the path is invalid or can't be a directory. + """ + try: + validate_path_could_be_dir(logs_dir) + except ValueError as ex: + raise ValueError(f"{component_name} logs directory is invalid: {ex}") From f5ea3e2716e74509442ecd99356943399aeb9311 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 17 Oct 2025 22:55:26 -0400 Subject: [PATCH 190/238] Replace EnvironmentError with OSError as per Ruff os-error-alias (UP024) --- .../clp_package_utils/controller.py | 2 +- .../clp-package-utils/clp_package_utils/general.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index f3ed95c172..33d863fb15 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -602,7 +602,7 @@ def stop(self) -> None: check_docker_dependencies( should_compose_project_be_running=True, project_name=self._project_name ) - except EnvironmentError as e: + except OSError as e: logger.warning( 'Docker dependencies check failed: "%s". Attempting to stop CLP containers ' "anyway...", diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index cb8ef8a367..fc0a8996a4 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -130,8 +130,8 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n :param should_compose_project_be_running: :param project_name: The Docker Compose project name to check. - :raises EnvironmentError: If any Docker dependency is not installed or the Docker Compose - project state doesn't match `should_compose_run`. + :raises OSError: If any Docker dependency is not installed or the Docker Compose project state + doesn't match `should_compose_run`. """ try: subprocess.run( @@ -142,13 +142,13 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n check=True, ) except subprocess.CalledProcessError: - raise EnvironmentError("docker is not installed or available on the path") + raise OSError("docker is not installed or available on the path") is_running = _is_docker_compose_project_running(project_name) if should_compose_project_be_running and not is_running: - raise EnvironmentError(f"Docker Compose project '{project_name}' is not running.") + raise OSError(f"Docker Compose project '{project_name}' is not running.") if not should_compose_project_be_running and is_running: - raise EnvironmentError("Docker Compose project '{project_name}' is already running.") + raise OSError("Docker Compose project '{project_name}' is already running.") def validate_port(port_name: str, hostname: str, port: int): @@ -697,7 +697,7 @@ def _is_docker_compose_project_running(project_name: str) -> bool: :param project_name: :return: Whether at least one instance is running. - :raises EnvironmentError: If Docker Compose is not installed or fails. + :raises OSError: If Docker Compose is not installed or fails. """ cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] try: @@ -705,7 +705,7 @@ def _is_docker_compose_project_running(project_name: str) -> bool: running_instances = json.loads(output) return len(running_instances) >= 1 except subprocess.CalledProcessError: - raise EnvironmentError("Docker Compose is not installed or not functioning properly.") + raise OSError("Docker Compose is not installed or not functioning properly.") def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: From 22593d890dcdb0977141acba084a5eeca43ee8ca Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 18 Oct 2025 09:05:25 -0400 Subject: [PATCH 191/238] refactor(general): Improve error handling for Docker and Docker Compose --- .../clp_package_utils/general.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index fc0a8996a4..363e4d4bf1 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -141,14 +141,17 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n stderr=subprocess.STDOUT, check=True, ) - except subprocess.CalledProcessError: - raise OSError("docker is not installed or available on the path") + except subprocess.CalledProcessError as e: + err_msg = "docker is not installed or available on the path" + raise OSError(err_msg) from e is_running = _is_docker_compose_project_running(project_name) if should_compose_project_be_running and not is_running: - raise OSError(f"Docker Compose project '{project_name}' is not running.") + err_msg = f"Docker Compose project '{project_name}' is not running." + raise OSError(err_msg) if not should_compose_project_be_running and is_running: - raise OSError("Docker Compose project '{project_name}' is already running.") + err_msg = "Docker Compose project '{project_name}' is already running." + raise OSError(err_msg) def validate_port(port_name: str, hostname: str, port: int): @@ -704,8 +707,9 @@ def _is_docker_compose_project_running(project_name: str) -> bool: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) running_instances = json.loads(output) return len(running_instances) >= 1 - except subprocess.CalledProcessError: - raise OSError("Docker Compose is not installed or not functioning properly.") + except subprocess.CalledProcessError as e: + err_msg = "Docker Compose is not installed or not functioning properly." + raise OSError(err_msg) from e def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: From bc69b26d4a001e95e8e182d889592b13282f95cc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 18 Oct 2025 09:06:27 -0400 Subject: [PATCH 192/238] refactor(general): Update dump_container_config call to remove return statement --- components/clp-package-utils/clp_package_utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 363e4d4bf1..80aa19b8cd 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -329,7 +329,7 @@ def dump_shared_container_config(container_clp_config: CLPConfig, clp_config: CL :param container_clp_config: :param clp_config: """ - return dump_container_config(container_clp_config, clp_config, CLP_SHARED_CONFIG_FILENAME) + dump_container_config(container_clp_config, clp_config, CLP_SHARED_CONFIG_FILENAME) def generate_container_start_cmd( From 1d2ab61b76cbccdd98b1901bdd0cc03f5b1639af Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 18 Oct 2025 09:10:30 -0400 Subject: [PATCH 193/238] refactor(docs): Update exception documentation to use 'raise' format --- .../clp-package-utils/clp_package_utils/controller.py | 2 +- components/clp-package-utils/clp_package_utils/general.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 33d863fb15..676348c00c 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -544,7 +544,7 @@ def _update_settings_object( :param parent_key_prefix: The prefix for keys at this level in the settings dictionary. :param settings: The settings to update. :param updates: The updates. - :raises ValueError: If a key in `updates` doesn't exist in `settings`. + :raise ValueError: If a key in `updates` doesn't exist in `settings`. """ for key, value in updates.items(): if key not in settings: diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 80aa19b8cd..dc33f803c0 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -130,7 +130,7 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n :param should_compose_project_be_running: :param project_name: The Docker Compose project name to check. - :raises OSError: If any Docker dependency is not installed or the Docker Compose project state + :raise OSError: If any Docker dependency is not installed or the Docker Compose project state doesn't match `should_compose_run`. """ try: @@ -700,7 +700,7 @@ def _is_docker_compose_project_running(project_name: str) -> bool: :param project_name: :return: Whether at least one instance is running. - :raises OSError: If Docker Compose is not installed or fails. + :raise OSError: If Docker Compose is not installed or fails. """ cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] try: @@ -725,7 +725,7 @@ def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): :param logs_dir: :param component_name: - :raises ValueError: If the path is invalid or can't be a directory. + :raise ValueError: If the path is invalid or can't be a directory. """ try: validate_path_could_be_dir(logs_dir) From d4c921ade727d59fb1ec8baae0f484e12783b22e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 18 Oct 2025 09:22:52 -0400 Subject: [PATCH 194/238] add missing `f` in f-string --- components/clp-package-utils/clp_package_utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index dc33f803c0..2df2e86be5 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -150,7 +150,7 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n err_msg = f"Docker Compose project '{project_name}' is not running." raise OSError(err_msg) if not should_compose_project_be_running and is_running: - err_msg = "Docker Compose project '{project_name}' is already running." + err_msg = f"Docker Compose project '{project_name}' is already running." raise OSError(err_msg) From e6bb002de031e2152480f8d87e6661724f30ab32 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 20 Oct 2025 02:13:30 -0400 Subject: [PATCH 195/238] Add retention periods check for garbage collector and disable the service's launch if none is configured --- .../clp-package-utils/clp_package_utils/controller.py | 11 +++++++++++ tools/deployment/package/docker-compose.base.yaml | 3 +++ 2 files changed, 14 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 676348c00c..1b7d8f5cbd 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -43,6 +43,7 @@ dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, + is_retention_period_configured, validate_db_config, validate_queue_config, validate_redis_config, @@ -505,6 +506,16 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: :return: Dictionary of environment variables necessary to launch the component. """ component_name = GARBAGE_COLLECTOR_COMPONENT_NAME + if not is_retention_period_configured(self._clp_config): + logger.info( + f"Retention period is not configured, skipping {component_name} creation..." + ) + return EnvVarsDict( + { + "CLP_GARBAGE_COLLECTOR_ENABLED": "0", + } + ) + logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 0663fba97b..38d5090510 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -342,6 +342,9 @@ ${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" garbage-collector: <<: *service_defaults hostname: "garbage_collector" + deploy: + # Value must be either 0 or 1. Set to 0 to disable the garbage collector. + replicas: "${CLP_GARBAGE_COLLECTOR_ENABLED:-1}" environment: CLP_DB_PASS: "${CLP_DB_PASS:?error}" CLP_DB_USER: "${CLP_DB_USER:?error}" From b0d0bccb5918490e14f4ef6769bf34441771e787 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 05:57:51 -0400 Subject: [PATCH 196/238] refactor(clp-package-utils): Remove unused `pass` statements from abstract methods. --- components/clp-package-utils/clp_package_utils/controller.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 1b7d8f5cbd..894153f304 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -87,14 +87,12 @@ def start(self) -> None: """ Starts the components. """ - pass @abstractmethod def stop(self) -> None: """ Stops the components. """ - pass @abstractmethod def _set_up_env(self) -> None: @@ -102,7 +100,6 @@ def _set_up_env(self) -> None: Sets up all components to run by preparing environment variables, directories, and configuration files. """ - pass def _set_up_env_for_database(self) -> EnvVarsDict: """ From 955a0ac0f3e7ae6fc9d2041f6426636661aa25a2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 06:07:50 -0400 Subject: [PATCH 197/238] refactor(clp-config): Remove unused typing import 'Set'. --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index ce2de4a3e5..c74389a106 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,7 +1,7 @@ import os import pathlib from enum import auto -from typing import Annotated, Any, ClassVar, Literal, Optional, Set, Union +from typing import Annotated, Any, ClassVar, Literal, Optional, Union from pydantic import ( BaseModel, From 106fdda0aa6ad80c257a64124ad926edb55f5aaa Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 06:09:10 -0400 Subject: [PATCH 198/238] apply doc suggestions --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index c74389a106..572b555845 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -778,7 +778,7 @@ def validate_presto_config(self): def transform_for_container(self): """ - Converts all relevant directories to absolute paths inside the container and updates + Converts all relevant directories to absolute paths inside the container, and updates component hostnames and ports to their container service names and default ports. """ self.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH From 7a1ea6e957e7145fd94489b95ab6defd079740ba Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 06:18:54 -0400 Subject: [PATCH 199/238] refactor(docker-compose): Update environment variable error messages for clarity. --- .../package/docker-compose.base.yaml | 42 +++++++++---------- tools/deployment/package/docker-compose.yaml | 12 +++--- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 38d5090510..e98eccd70a 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -66,9 +66,9 @@ services: user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: MYSQL_DATABASE: "${CLP_DB_NAME:-clp-db}" - MYSQL_PASSWORD: "${CLP_DB_PASS:?error}" - MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS:?error}" - MYSQL_USER: "${CLP_DB_USER:?error}" + MYSQL_PASSWORD: "${CLP_DB_PASS:?Please set a value.}" + MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS:?Please set a value.}" + MYSQL_USER: "${CLP_DB_USER:?Please set a value.}" ports: - host_ip: "${CLP_DB_HOST:-127.0.0.1}" published: "${CLP_DB_PORT:-3306}" @@ -91,16 +91,16 @@ services: "mysqladmin", "ping", "--silent", "-h", "127.0.0.1", - "-u", "${CLP_DB_USER:?error}", - "--password=${CLP_DB_PASS:?error}" + "-u", "${CLP_DB_USER:?Please set a value.}", + "--password=${CLP_DB_PASS:?Please set a value.}" ] db-table-creator: <<: *service_defaults hostname: "db_table_creator" environment: - CLP_DB_PASS: "${CLP_DB_PASS:?error}" - CLP_DB_USER: "${CLP_DB_USER:?error}" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - *volume_clp_config_readonly @@ -121,8 +121,8 @@ services: hostname: "queue" user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: - RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS:?error}" - RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER:?error}" + RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS:?Please set a value.}" + RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER:?Please set a value.}" RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" ports: - host_ip: "${CLP_QUEUE_HOST:-127.0.0.1}" @@ -162,7 +162,7 @@ services: command: [ "redis-server", "/usr/local/etc/redis/redis.conf", - "--requirepass", "${CLP_REDIS_PASS:?error}" + "--requirepass", "${CLP_REDIS_PASS:?Please set a value.}" ] healthcheck: <<: *healthcheck_defaults @@ -171,7 +171,7 @@ services: "redis-cli", "-h", "127.0.0.1", "-p", "6379", - "-a", "${CLP_REDIS_PASS:?error}", + "-a", "${CLP_REDIS_PASS:?Please set a value.}", "PING" ] @@ -227,14 +227,14 @@ services: hostname: "compression_scheduler" stop_grace_period: "300s" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" - CLP_DB_PASS: "${CLP_DB_PASS:?error}" - CLP_DB_USER: "${CLP_DB_USER:?error}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} + redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly @@ -259,7 +259,7 @@ services: hostname: "compression_worker" stop_grace_period: "60s" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" @@ -267,7 +267,7 @@ services: CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} + redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly @@ -299,8 +299,8 @@ ${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" environment: AWS_ACCESS_KEY_ID: "${CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID:-}" AWS_SECRET_ACCESS_KEY: "${CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY:-}" - CLP_DB_PASS: "${CLP_DB_PASS:?error}" - CLP_DB_USER: "${CLP_DB_USER:?error}" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" HOST: "0.0.0.0" NODE_ENV: "production" NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" @@ -346,8 +346,8 @@ ${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" # Value must be either 0 or 1. Set to 0 to disable the garbage collector. replicas: "${CLP_GARBAGE_COLLECTOR_ENABLED:-1}" environment: - CLP_DB_PASS: "${CLP_DB_PASS:?error}" - CLP_DB_USER: "${CLP_DB_USER:?error}" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/garbage_collector" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index e227f3ab3b..bc731e6fe7 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -37,14 +37,14 @@ services: <<: *service_defaults hostname: "query_scheduler" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" - CLP_DB_PASS: "${CLP_DB_PASS:?error}" - CLP_DB_USER: "${CLP_DB_USER:?error}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_clp_config_readonly - *volume_clp_logs @@ -74,7 +74,7 @@ services: <<: *service_defaults hostname: "query_worker" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" @@ -82,7 +82,7 @@ services: CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly From a732939be9c115e6be9784fc3f01f3c3a822648a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 06:50:16 -0400 Subject: [PATCH 200/238] Adjust volume bindings to fix directory mappings when CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST/CLP_ARCHIVE_OUTPUT_DIR_HOST is undefined --- tools/deployment/package/docker-compose.base.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index e98eccd70a..927414f930 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -274,9 +274,9 @@ services: - *volume_clp_logs - *volume_logs_input_readonly - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ -${CLP_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/archives" +${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ -${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" +${CLP_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/staged-archives" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" From c687f7839bc82f332e10922059c8a82bc4c9d184 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 06:51:00 -0400 Subject: [PATCH 201/238] fix string indents --- tools/deployment/package/docker-compose.base.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 927414f930..8ef789711e 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -273,10 +273,10 @@ services: - *volume_clp_config_readonly - *volume_clp_logs - *volume_logs_input_readonly - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ -${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" - - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ -${CLP_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/staged-archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ + :${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" + - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ + :${CLP_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/staged-archives" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" From 19e8a9d64863cd883dcfd3245f04dd3fa92ca922 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 06:52:12 -0400 Subject: [PATCH 202/238] apply doc suggestions --- tools/deployment/package/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index bc731e6fe7..8eb0fe79e6 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -2,7 +2,7 @@ name: "clp-package" include: ["docker-compose.base.yaml"] -# Below x-* definitions are duplicated from docker-compose.base.yaml. Refer to that file for +# The `x-*` definitions below are duplicated from docker-compose.base.yaml. Refer to that file for # documentation. x-service-defaults: &service_defaults image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" From c53c0f1961f0a52e5c4a870fdc6a37709b71da3a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:30:58 -0400 Subject: [PATCH 203/238] apply doc suggestions --- .../clp_package_utils/controller.py | 60 +++++++++++-------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 894153f304..35987e0cca 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -120,18 +120,19 @@ def _set_up_env_for_database(self) -> EnvVarsDict: _chown_paths_if_root(data_dir, logs_dir) env_vars = EnvVarsDict() - # Connection + + # Connection params env_vars |= { "CLP_DB_HOST": _get_ip_from_hostname(self._clp_config.database.host), "CLP_DB_NAME": self._clp_config.database.name, "CLP_DB_PORT": str(self._clp_config.database.port), } - # Credential + # Credentials env_vars |= { "CLP_DB_PASS": self._clp_config.database.password, "CLP_DB_USER": self._clp_config.database.username, } - # Path + # Paths env_vars |= { "CLP_DB_CONF_LOGGING_FILE_HOST": str(conf_logging_file), "CLP_DB_DATA_DIR_HOST": str(data_dir), @@ -162,17 +163,18 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: _chown_paths_if_root(logs_dir) env_vars = EnvVarsDict() - # Connection + + # Connection params env_vars |= { "CLP_QUEUE_HOST": _get_ip_from_hostname(self._clp_config.queue.host), "CLP_QUEUE_PORT": str(self._clp_config.queue.port), } - # Credential + # Credentials env_vars |= { "CLP_QUEUE_PASS": self._clp_config.queue.password, "CLP_QUEUE_USER": self._clp_config.queue.username, } - # Path + # Paths env_vars |= { "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), } @@ -198,23 +200,24 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: _chown_paths_if_root(data_dir, logs_dir) env_vars = EnvVarsDict() - # Backend + + # Backend databases env_vars |= { "CLP_REDIS_BACKEND_DB_COMPRESSION": str( self._clp_config.redis.compression_backend_database ), "CLP_REDIS_BACKEND_DB_QUERY": str(self._clp_config.redis.query_backend_database), } - # Connection + # Connection params env_vars |= { "CLP_REDIS_HOST": _get_ip_from_hostname(self._clp_config.redis.host), "CLP_REDIS_PORT": str(self._clp_config.redis.port), } - # Credential + # Credentials env_vars |= { "CLP_REDIS_PASS": self._clp_config.redis.password, } - # Path + # Paths env_vars |= { "CLP_REDIS_CONF_FILE_HOST": str(conf_file), "CLP_REDIS_DATA_DIR_HOST": str(data_dir), @@ -242,19 +245,20 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: _chown_paths_if_root(data_dir, logs_dir) env_vars = EnvVarsDict() - # Collection + + # Collections env_vars |= { "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": ( self._clp_config.results_cache.stream_collection_name ), } - # Connection + # Connection params env_vars |= { "CLP_RESULTS_CACHE_DB_NAME": self._clp_config.results_cache.db_name, "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self._clp_config.results_cache.host), "CLP_RESULTS_CACHE_PORT": str(self._clp_config.results_cache.port), } - # Path + # Paths env_vars |= { "CLP_RESULTS_CACHE_CONF_FILE_HOST": str(conf_file), "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), @@ -273,6 +277,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") env_vars = EnvVarsDict() + # Logging env_vars |= { "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": ( @@ -292,6 +297,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") env_vars = EnvVarsDict() + # Logging env_vars |= { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self._clp_config.query_scheduler.logging_level, @@ -313,13 +319,14 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() + # Logging env_vars |= { "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": ( self._clp_config.compression_worker.logging_level ), } - # Resource + # Resources env_vars |= { "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_workers), } @@ -340,11 +347,12 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() + # Logging env_vars |= { "CLP_QUERY_WORKER_LOGGING_LEVEL": self._clp_config.query_worker.logging_level, } - # Resource + # Resources env_vars |= { "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } @@ -365,11 +373,12 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() + # Logging env_vars |= { "CLP_REDUCER_LOGGING_LEVEL": self._clp_config.reducer.logging_level, } - # Resource + # Resources env_vars |= { "CLP_REDUCER_CONCURRENCY": str(num_workers), "CLP_REDUCER_UPSERT_INTERVAL": str(self._clp_config.reducer.upsert_interval), @@ -484,7 +493,8 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: settings_json_file.write(json.dumps(server_settings_json)) env_vars = EnvVarsDict() - # Connection + + # Connection params env_vars |= { "CLP_WEBUI_HOST": _get_ip_from_hostname(self._clp_config.webui.host), "CLP_WEBUI_PORT": str(self._clp_config.webui.port), @@ -644,7 +654,7 @@ def _set_up_env(self) -> None: # Prepare environment variables for all components. env_vars = EnvVarsDict() - # Credential + # Credentials if self._clp_config.stream_output.storage.type == StorageType.S3: stream_output_aws_auth = ( self._clp_config.stream_output.storage.s3_config.aws_authentication @@ -659,7 +669,7 @@ def _set_up_env(self) -> None: ), } - # Identity + # Identity config env_vars |= { "CLP_FIRST_PARTY_SERVICE_UID_GID": DEFAULT_UID_GID, "CLP_THIRD_PARTY_SERVICE_UID_GID": ( @@ -667,13 +677,13 @@ def _set_up_env(self) -> None: ), } - # Package + # Package config env_vars |= { "CLP_PACKAGE_CONTAINER_IMAGE_REF": self._clp_config.container_image_ref, "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, } - # Path + # Paths aws_config_dir = self._clp_config.aws_config_directory env_vars |= { # General @@ -682,13 +692,15 @@ def _set_up_env(self) -> None: # Config "CLP_AWS_CONFIG_DIR_HOST": (None if aws_config_dir is None else str(aws_config_dir)), } - # Input + + # Input config if self._clp_config.logs_input.type == StorageType.FS: env_vars["CLP_LOGS_INPUT_DIR_CONTAINER"] = str( container_clp_config.logs_input.directory ) env_vars["CLP_LOGS_INPUT_DIR_HOST"] = str(self._clp_config.logs_input.directory) - # Output + + # Output config archive_output_dir_str = str(self._clp_config.archive_output.get_directory()) stream_output_dir_str = str(self._clp_config.stream_output.get_directory()) if self._clp_config.archive_output.storage.type == StorageType.FS: @@ -700,7 +712,7 @@ def _set_up_env(self) -> None: if self._clp_config.stream_output.storage.type == StorageType.S3: env_vars["CLP_STAGED_STREAM_OUTPUT_DIR_HOST"] = stream_output_dir_str - # Component-specific + # Component-specific config env_vars |= self._set_up_env_for_database() env_vars |= self._set_up_env_for_queue() env_vars |= self._set_up_env_for_redis() From a2fa02c8964d76d0ece0eef0ca45e3a5b651cfa6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:31:10 -0400 Subject: [PATCH 204/238] fix(clp-package-utils): Remove extra space in `.env` file comment. --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 35987e0cca..395157e320 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -725,7 +725,7 @@ def _set_up_env(self) -> None: env_vars |= self._set_up_env_for_webui(container_clp_config) env_vars |= self._set_up_env_for_garbage_collector() - # Write the environment variables to the `.env ` file. + # Write the environment variables to the `.env` file. with open(f"{self._clp_home}/.env", "w") as env_file: for key, value in env_vars.items(): if value is None: From 05810bec3b459cf089580abb59aa2ee3971f3b27 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:32:09 -0400 Subject: [PATCH 205/238] apply doc suggestions --- components/clp-package-utils/clp_package_utils/controller.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 395157e320..ea5be2ae4f 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -686,11 +686,9 @@ def _set_up_env(self) -> None: # Paths aws_config_dir = self._clp_config.aws_config_directory env_vars |= { - # General + "CLP_AWS_CONFIG_DIR_HOST": (None if aws_config_dir is None else str(aws_config_dir)), "CLP_DATA_DIR_HOST": str(self._clp_config.data_directory), "CLP_LOGS_DIR_HOST": str(self._clp_config.logs_directory), - # Config - "CLP_AWS_CONFIG_DIR_HOST": (None if aws_config_dir is None else str(aws_config_dir)), } # Input config From 2a02ecb0d21d1d5b9c799bd5d9700d7907120608 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:32:54 -0400 Subject: [PATCH 206/238] apply doc suggestions --- components/clp-package-utils/clp_package_utils/controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index ea5be2ae4f..9743fa4803 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -651,7 +651,6 @@ def _set_up_env(self) -> None: num_workers = self._get_num_workers() dump_shared_container_config(container_clp_config, self._clp_config) - # Prepare environment variables for all components. env_vars = EnvVarsDict() # Credentials From 678de80eac42f986fb1d2ca1fd448a9104b9d115 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:34:36 -0400 Subject: [PATCH 207/238] Update comment for `_get_num_workers` method - apply doc suggestion --- components/clp-package-utils/clp_package_utils/controller.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 9743fa4803..76697bad65 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -639,10 +639,9 @@ def stop(self) -> None: @staticmethod def _get_num_workers() -> int: """ - TODO: Revisit after moving from single-container to multi-container workers. See issue - @y-scope/clp#1424 for details. :return: Number of worker processes to run. """ + # This will change when we move from single to multi-container workers. See y-scope/clp#1424 return multiprocessing.cpu_count() // 2 def _set_up_env(self) -> None: From c30adaec998e6749c86a2d2e5a5dc8c08ee08296 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:35:29 -0400 Subject: [PATCH 208/238] Simplify log message for CLP start action - apply suggestion --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 76697bad65..2f049e56f8 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -608,7 +608,7 @@ def start(self) -> None: cwd=self._clp_home, check=True, ) - logger.info("CLP is started.") + logger.info("CLP started.") def stop(self) -> None: """ From 07d1e3512aefd3b9d9dd1a2723efe52855070ba2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:37:55 -0400 Subject: [PATCH 209/238] Update comments for consistency in `controller.py`. --- .../clp_package_utils/controller.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 2f049e56f8..de4390de91 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -138,7 +138,7 @@ def _set_up_env_for_database(self) -> EnvVarsDict: "CLP_DB_DATA_DIR_HOST": str(data_dir), "CLP_DB_LOGS_DIR_HOST": str(logs_dir), } - # Runtime + # Runtime params env_vars |= { "CLP_DB_IMAGE": ( "mysql:8.0.23" if self._clp_config.database.type == "mysql" else "mariadb:10-jammy" @@ -278,7 +278,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging + # Logging params env_vars |= { "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": ( self._clp_config.compression_scheduler.logging_level @@ -298,7 +298,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging + # Logging params env_vars |= { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self._clp_config.query_scheduler.logging_level, } @@ -320,7 +320,7 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging + # Logging params env_vars |= { "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": ( self._clp_config.compression_worker.logging_level @@ -348,7 +348,7 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging + # Logging params env_vars |= { "CLP_QUERY_WORKER_LOGGING_LEVEL": self._clp_config.query_worker.logging_level, } @@ -374,7 +374,7 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging + # Logging params env_vars |= { "CLP_REDUCER_LOGGING_LEVEL": self._clp_config.reducer.logging_level, } @@ -499,7 +499,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: "CLP_WEBUI_HOST": _get_ip_from_hostname(self._clp_config.webui.host), "CLP_WEBUI_PORT": str(self._clp_config.webui.port), } - # Security + # Security params env_vars |= { "CLP_WEBUI_RATE_LIMIT": str(self._clp_config.webui.rate_limit), } @@ -530,7 +530,7 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging + # Logging params env_vars |= {"CLP_GC_LOGGING_LEVEL": self._clp_config.garbage_collector.logging_level} return env_vars From 0bf6b70fbb3cab4563e3ba9ba8075c74cf3ae3ff Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 11:18:57 -0400 Subject: [PATCH 210/238] feat(clp-package-utils): Add custom exceptions for Docker dependency errors; do not attempt to stop CLP when DockerComposeProjectNotRunningError is hit --- .../clp_package_utils/controller.py | 11 +++- .../clp_package_utils/general.py | 66 +++++++++++++++---- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index de4390de91..b3c5313f70 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -40,6 +40,7 @@ from clp_package_utils.general import ( check_docker_dependencies, CONTAINER_CLP_HOME, + DockerComposeProjectNotRunningError, dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, @@ -49,6 +50,7 @@ validate_redis_config, validate_results_cache_config, validate_webui_config, + DockerDependencyError, ) LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH @@ -589,6 +591,7 @@ def start(self) -> None: """ Starts CLP's components using Docker Compose. + :raise: Propagates `check_docker_dependencies`'s exceptions. :raise: Propagates `subprocess.run`'s exceptions. """ check_docker_dependencies( @@ -620,7 +623,13 @@ def stop(self) -> None: check_docker_dependencies( should_compose_project_be_running=True, project_name=self._project_name ) - except OSError as e: + except DockerComposeProjectNotRunningError: + logger.info( + "Docker Compose project '%s' is not running. Nothing to stop.", + self._project_name, + ) + return + except DockerDependencyError as e: logger.warning( 'Docker dependencies check failed: "%s". Attempting to stop CLP containers ' "anyway...", diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 2df2e86be5..6d769ee79a 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -49,6 +49,47 @@ DOCKER_MOUNT_TYPE_STRINGS = ["bind"] +class DockerDependencyError(OSError): + """Base class for errors related to Docker dependencies.""" + + +class DockerNotAvailableError(DockerDependencyError): + """Raised when Docker or Docker Compose is unavailable.""" + + def __init__(self, base_message: str, process_error: subprocess.CalledProcessError): + message = base_message + output_chunks: list[str] = [] + for stream in (process_error.output, process_error.stderr): + if stream is None: + continue + if isinstance(stream, bytes): + text = stream.decode(errors="replace") + else: + text = str(stream) + text = text.strip() + if text: + output_chunks.append(text) + if len(output_chunks) > 0: + message = f"{base_message}\n" + "\n".join(output_chunks) + super().__init__(errno.ENOENT, message) + + +class DockerComposeProjectNotRunningError(DockerDependencyError): + """Raised when an expected Docker Compose project is not running.""" + + def __init__(self, project_name: str): + super().__init__(errno.ESRCH, f"Docker Compose project '{project_name}' is not running.") + + +class DockerComposeProjectAlreadyRunningError(DockerDependencyError): + """Raised when a Docker Compose project is already running but should not be.""" + + def __init__(self, project_name: str): + super().__init__( + errno.EEXIST, f"Docker Compose project '{project_name}' is already running." + ) + + class DockerMountType(enum.IntEnum): BIND = 0 @@ -130,8 +171,9 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n :param should_compose_project_be_running: :param project_name: The Docker Compose project name to check. - :raise OSError: If any Docker dependency is not installed or the Docker Compose project state - doesn't match `should_compose_run`. + :raise DockerNotAvailableError: If any Docker dependency is not installed. + :raise DockerComposeProjectNotRunningError: If the project isn't running when it should be. + :raise DockerComposeProjectAlreadyRunningError: If the project is running when it shouldn't be. """ try: subprocess.run( @@ -142,16 +184,16 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n check=True, ) except subprocess.CalledProcessError as e: - err_msg = "docker is not installed or available on the path" - raise OSError(err_msg) from e + raise DockerNotAvailableError( + "docker is not installed or available on the path", e + ) from e is_running = _is_docker_compose_project_running(project_name) + if should_compose_project_be_running and not is_running: - err_msg = f"Docker Compose project '{project_name}' is not running." - raise OSError(err_msg) + raise DockerComposeProjectNotRunningError(project_name) if not should_compose_project_be_running and is_running: - err_msg = f"Docker Compose project '{project_name}' is already running." - raise OSError(err_msg) + raise DockerComposeProjectAlreadyRunningError(project_name) def validate_port(port_name: str, hostname: str, port: int): @@ -700,7 +742,8 @@ def _is_docker_compose_project_running(project_name: str) -> bool: :param project_name: :return: Whether at least one instance is running. - :raise OSError: If Docker Compose is not installed or fails. + :raise DockerNotAvailableError: If Docker Compose is not installed or fails. The error message + includes Docker's command output when available. """ cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] try: @@ -708,8 +751,9 @@ def _is_docker_compose_project_running(project_name: str) -> bool: running_instances = json.loads(output) return len(running_instances) >= 1 except subprocess.CalledProcessError as e: - err_msg = "Docker Compose is not installed or not functioning properly." - raise OSError(err_msg) from e + raise DockerNotAvailableError( + "Docker Compose is not installed or not functioning properly.", e + ) from e def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: From 2986082589779cd037cbe7fddb36aaf2321ecbe7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 11:23:42 -0400 Subject: [PATCH 211/238] fix(clp-package-utils): Use `docker --version` to check docker availability; Replace `subprocess.run` with `subprocess.check_output` for Docker check. --- components/clp-package-utils/clp_package_utils/general.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 6d769ee79a..58ff00ea6b 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -176,12 +176,9 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n :raise DockerComposeProjectAlreadyRunningError: If the project is running when it shouldn't be. """ try: - subprocess.run( - "command -v docker", - shell=True, - stdout=subprocess.DEVNULL, + subprocess.check_output( + ["docker", "--version"], stderr=subprocess.STDOUT, - check=True, ) except subprocess.CalledProcessError as e: raise DockerNotAvailableError( From e499443df01a084ba34dce3696c5e32a465ee83f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 11:28:59 -0400 Subject: [PATCH 212/238] Avoid mapping ~/.aws on host when CLP_AWS_CONFIG_DIR_HOST is undefined --- tools/deployment/package/docker-compose.base.yaml | 9 ++------- tools/deployment/package/docker-compose.yaml | 7 +------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 8ef789711e..542bc31adb 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -30,11 +30,6 @@ x-healthcheck-defaults: &healthcheck_defaults # Common volume definitions. x-volume-definitions: - aws-config-readonly: &volume_aws_config_readonly - type: "bind" - source: "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}" - target: "/.aws" - read_only: true clp-config-readonly: &volume_clp_config_readonly type: "bind" source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" @@ -236,10 +231,10 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - *volume_logs_input_readonly + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -269,7 +264,6 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - *volume_logs_input_readonly @@ -277,6 +271,7 @@ services: :${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ :${CLP_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/staged-archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 8eb0fe79e6..bc283ed7cd 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -17,11 +17,6 @@ x-healthcheck-defaults: &healthcheck_defaults start_period: "60s" timeout: "2s" x-volume-definitions: - aws-config-readonly: &volume_aws_config_readonly - type: "bind" - source: "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}" - target: "/.aws" - read_only: true clp-config-readonly: &volume_clp_config_readonly type: "bind" source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" @@ -84,10 +79,10 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" - "${CLP_STAGED_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/staged-streams" command: [ From b788c3350cdb4bf6f1f3152fe2184b9c4c841392 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 11:30:23 -0400 Subject: [PATCH 213/238] alphabetize mounts --- tools/deployment/package/docker-compose.base.yaml | 2 +- tools/deployment/package/docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 542bc31adb..75594e6d29 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -267,11 +267,11 @@ services: - *volume_clp_config_readonly - *volume_clp_logs - *volume_logs_input_readonly + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ :${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ :${CLP_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/staged-archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index bc283ed7cd..b804c015e0 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -83,8 +83,8 @@ services: - *volume_clp_logs - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" - "${CLP_STAGED_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/staged-streams" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" command: [ "python3", "-u", From 6fc219cbfc6c7bd10b7998dc25112608755cc58b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 11:40:25 -0400 Subject: [PATCH 214/238] Do not map logs input dir when CLP_LOGS_INPUT_DIR_HOST is undefined --- tools/deployment/package/docker-compose.base.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 75594e6d29..03a6aabb6e 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -39,11 +39,6 @@ x-volume-definitions: type: "bind" source: "${CLP_LOGS_DIR_HOST:-./var/log}" target: "/var/log" - logs-input-readonly: &volume_logs_input_readonly - type: "bind" - source: "${CLP_LOGS_INPUT_DIR_HOST:-/}" - target: "${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" - read_only: true volumes: # Dummy volume to use when a bind mount is not desired. @@ -233,8 +228,8 @@ services: volumes: - *volume_clp_config_readonly - *volume_clp_logs - - *volume_logs_input_readonly - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" + - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -266,8 +261,8 @@ services: volumes: - *volume_clp_config_readonly - *volume_clp_logs - - *volume_logs_input_readonly - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" + - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ :${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ @@ -306,7 +301,7 @@ services: published: "${CLP_WEBUI_PORT:-4000}" target: 4000 volumes: - - *volume_aws_config_readonly + - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" - type: "bind" source: "./var/www/webui/client/settings.json" From 09261802904ea7e984eccfaa3e8dcaf255900d33 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 11:45:13 -0400 Subject: [PATCH 215/238] lint --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- components/clp-package-utils/clp_package_utils/general.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index b3c5313f70..92ab38a7a2 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -41,6 +41,7 @@ check_docker_dependencies, CONTAINER_CLP_HOME, DockerComposeProjectNotRunningError, + DockerDependencyError, dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, @@ -50,7 +51,6 @@ validate_redis_config, validate_results_cache_config, validate_webui_config, - DockerDependencyError, ) LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 58ff00ea6b..8dc3833da9 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -181,9 +181,7 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as e: - raise DockerNotAvailableError( - "docker is not installed or available on the path", e - ) from e + raise DockerNotAvailableError("docker is not installed or available on the path", e) from e is_running = _is_docker_compose_project_running(project_name) From e2daf327c404a3f984e228fafba85c70dffcb8d3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 12:09:15 -0400 Subject: [PATCH 216/238] lint --- tools/deployment/package/docker-compose.base.yaml | 14 ++++++++------ tools/deployment/package/docker-compose.yaml | 14 ++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 03a6aabb6e..35370f117b 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -217,14 +217,15 @@ services: hostname: "compression_scheduler" stop_grace_period: "300s" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:\ + ${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\ + /${CLP_REDIS_BACKEND_DB_COMPRESSION:-1}" volumes: - *volume_clp_config_readonly - *volume_clp_logs @@ -249,15 +250,16 @@ services: hostname: "compression_worker" stop_grace_period: "60s" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}\ + :${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/compression_worker" CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\ + /${CLP_REDIS_BACKEND_DB_COMPRESSION:-1}" volumes: - *volume_clp_config_readonly - *volume_clp_logs diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index b804c015e0..8674218473 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -32,14 +32,15 @@ services: <<: *service_defaults hostname: "query_scheduler" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:\ + ${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\ + /${CLP_REDIS_BACKEND_DB_QUERY:-0}" volumes: - *volume_clp_config_readonly - *volume_clp_logs @@ -69,15 +70,16 @@ services: <<: *service_defaults hostname: "query_worker" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:\ + ${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/query_worker" CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\ + /${CLP_REDIS_BACKEND_DB_QUERY:-0}" volumes: - *volume_clp_config_readonly - *volume_clp_logs From 49561062937137d4ebac10641f34cffae54eee43 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 12:18:56 -0400 Subject: [PATCH 217/238] fix: Add AWS config directory mount to garbage collector --- tools/deployment/package/docker-compose.base.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 35370f117b..f53d71f1e1 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -348,6 +348,7 @@ services: - *volume_clp_config_readonly - *volume_clp_logs - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" depends_on: db-table-creator: From 91cacb01891bda2ab30faa5bdd4aed10984ce864 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 12:27:50 -0400 Subject: [PATCH 218/238] fix(clp-package-utils): Add return type annotations to custom exception constructors. --- components/clp-package-utils/clp_package_utils/general.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 8dc3833da9..e98e0b9e0c 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -56,7 +56,7 @@ class DockerDependencyError(OSError): class DockerNotAvailableError(DockerDependencyError): """Raised when Docker or Docker Compose is unavailable.""" - def __init__(self, base_message: str, process_error: subprocess.CalledProcessError): + def __init__(self, base_message: str, process_error: subprocess.CalledProcessError) -> None: message = base_message output_chunks: list[str] = [] for stream in (process_error.output, process_error.stderr): @@ -77,14 +77,14 @@ def __init__(self, base_message: str, process_error: subprocess.CalledProcessErr class DockerComposeProjectNotRunningError(DockerDependencyError): """Raised when an expected Docker Compose project is not running.""" - def __init__(self, project_name: str): + def __init__(self, project_name: str) -> None: super().__init__(errno.ESRCH, f"Docker Compose project '{project_name}' is not running.") class DockerComposeProjectAlreadyRunningError(DockerDependencyError): """Raised when a Docker Compose project is already running but should not be.""" - def __init__(self, project_name: str): + def __init__(self, project_name: str) -> None: super().__init__( errno.EEXIST, f"Docker Compose project '{project_name}' is already running." ) From c2089420622323741528542a5171dca21dc16f8c Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 22 Oct 2025 07:33:31 -0400 Subject: [PATCH 219/238] Rename CLP_DB_IMAGE to CLP_DB_CONTAINER_IMAGE_REF. --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- tools/deployment/package/docker-compose.base.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 92ab38a7a2..daa36eaa1c 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -142,7 +142,7 @@ def _set_up_env_for_database(self) -> EnvVarsDict: } # Runtime params env_vars |= { - "CLP_DB_IMAGE": ( + "CLP_DB_CONTAINER_IMAGE_REF": ( "mysql:8.0.23" if self._clp_config.database.type == "mysql" else "mariadb:10-jammy" ), } diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index f53d71f1e1..fc8d93e6aa 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -51,7 +51,7 @@ volumes: services: database: <<: *service_defaults - image: "${CLP_DB_IMAGE:-mariadb:10-jammy}" + image: "${CLP_DB_CONTAINER_IMAGE_REF:-mariadb:10-jammy}" hostname: "database" user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: From 6cea88b42c49f72a70fe02a3e8ba1edab687ee09 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 22 Oct 2025 07:34:25 -0400 Subject: [PATCH 220/238] Simplify package start/stop logs. --- components/clp-package-utils/clp_package_utils/controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index daa36eaa1c..ea9752e0f9 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -611,7 +611,7 @@ def start(self) -> None: cwd=self._clp_home, check=True, ) - logger.info("CLP started.") + logger.info("Started CLP.") def stop(self) -> None: """ @@ -643,7 +643,7 @@ def stop(self) -> None: cwd=self._clp_home, check=True, ) - logger.info("All CLP containers stopped.") + logger.info("Stopped CLP.") @staticmethod def _get_num_workers() -> int: From 8c5a16735a44c0b955f6d31fc6823f1429fb993a Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 22 Oct 2025 07:35:51 -0400 Subject: [PATCH 221/238] Refactor new Docker exceptions. --- components/clp-package-utils/clp_package_utils/general.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index e98e0b9e0c..886486af1d 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -59,7 +59,7 @@ class DockerNotAvailableError(DockerDependencyError): def __init__(self, base_message: str, process_error: subprocess.CalledProcessError) -> None: message = base_message output_chunks: list[str] = [] - for stream in (process_error.output, process_error.stderr): + for stream in (process_error.stdout, process_error.stderr): if stream is None: continue if isinstance(stream, bytes): @@ -70,12 +70,12 @@ def __init__(self, base_message: str, process_error: subprocess.CalledProcessErr if text: output_chunks.append(text) if len(output_chunks) > 0: - message = f"{base_message}\n" + "\n".join(output_chunks) + message = "\n".join([base_message, *output_chunks]) super().__init__(errno.ENOENT, message) class DockerComposeProjectNotRunningError(DockerDependencyError): - """Raised when an expected Docker Compose project is not running.""" + """Raised when a Docker Compose project is not running but should be.""" def __init__(self, project_name: str) -> None: super().__init__(errno.ESRCH, f"Docker Compose project '{project_name}' is not running.") From f02aff4bae17a3e7b2c365b24e93708e9c3a783a Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 22 Oct 2025 07:36:26 -0400 Subject: [PATCH 222/238] general.py: Minor refactoring. --- components/clp-package-utils/clp_package_utils/general.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 886486af1d..cdbe0098e5 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -184,7 +184,6 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n raise DockerNotAvailableError("docker is not installed or available on the path", e) from e is_running = _is_docker_compose_project_running(project_name) - if should_compose_project_be_running and not is_running: raise DockerComposeProjectNotRunningError(project_name) if not should_compose_project_be_running and is_running: @@ -738,7 +737,7 @@ def _is_docker_compose_project_running(project_name: str) -> bool: :param project_name: :return: Whether at least one instance is running. :raise DockerNotAvailableError: If Docker Compose is not installed or fails. The error message - includes Docker's command output when available. + includes the Docker command's output when available. """ cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] try: From 667d75399aada34508d09116cd05b381e3b15d48 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 22 Oct 2025 07:38:37 -0400 Subject: [PATCH 223/238] Rename CLP_GC_LOGGING_LEVEL to CLP_GARBAGE_COLLECTOR_LOGGING_LEVEL. --- components/clp-package-utils/clp_package_utils/controller.py | 4 +++- tools/deployment/package/docker-compose.base.yaml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index ea9752e0f9..340518c848 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -533,7 +533,9 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: env_vars = EnvVarsDict() # Logging params - env_vars |= {"CLP_GC_LOGGING_LEVEL": self._clp_config.garbage_collector.logging_level} + env_vars |= { + "CLP_GARBAGE_COLLECTOR_LOGGING_LEVEL": self._clp_config.garbage_collector.logging_level + } return env_vars diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index fc8d93e6aa..73c41b20b3 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -341,7 +341,7 @@ services: CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_HOME: "/opt/clp" - CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" + CLP_LOGGING_LEVEL: "${CLP_GARBAGE_COLLECTOR_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/garbage_collector" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: From 930a2dd71937c79f9ff3b2dde265ebb35e402b78 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 22 Oct 2025 07:40:00 -0400 Subject: [PATCH 224/238] Put operator on next line of multiline statement. --- tools/deployment/package/docker-compose.base.yaml | 4 ++-- tools/deployment/package/docker-compose.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 73c41b20b3..937ea73c09 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -217,8 +217,8 @@ services: hostname: "compression_scheduler" stop_grace_period: "300s" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:\ - ${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}\ + :${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 8674218473..f81ea27280 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -32,8 +32,8 @@ services: <<: *service_defaults hostname: "query_scheduler" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:\ - ${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}\ + :${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" @@ -70,8 +70,8 @@ services: <<: *service_defaults hostname: "query_worker" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:\ - ${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}\ + :${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" From 340a5e776116996d399543c08e87d2a80405a6ee Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 22 Oct 2025 07:42:28 -0400 Subject: [PATCH 225/238] Use term 'config' instead of less common 'params'; Add required newline before comments. --- .../clp_package_utils/controller.py | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 340518c848..f0d96643e3 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -123,24 +123,27 @@ def _set_up_env_for_database(self) -> EnvVarsDict: env_vars = EnvVarsDict() - # Connection params + # Connection config env_vars |= { "CLP_DB_HOST": _get_ip_from_hostname(self._clp_config.database.host), "CLP_DB_NAME": self._clp_config.database.name, "CLP_DB_PORT": str(self._clp_config.database.port), } + # Credentials env_vars |= { "CLP_DB_PASS": self._clp_config.database.password, "CLP_DB_USER": self._clp_config.database.username, } + # Paths env_vars |= { "CLP_DB_CONF_LOGGING_FILE_HOST": str(conf_logging_file), "CLP_DB_DATA_DIR_HOST": str(data_dir), "CLP_DB_LOGS_DIR_HOST": str(logs_dir), } - # Runtime params + + # Runtime config env_vars |= { "CLP_DB_CONTAINER_IMAGE_REF": ( "mysql:8.0.23" if self._clp_config.database.type == "mysql" else "mariadb:10-jammy" @@ -166,16 +169,18 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: env_vars = EnvVarsDict() - # Connection params + # Connection config env_vars |= { "CLP_QUEUE_HOST": _get_ip_from_hostname(self._clp_config.queue.host), "CLP_QUEUE_PORT": str(self._clp_config.queue.port), } + # Credentials env_vars |= { "CLP_QUEUE_PASS": self._clp_config.queue.password, "CLP_QUEUE_USER": self._clp_config.queue.username, } + # Paths env_vars |= { "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), @@ -210,15 +215,18 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: ), "CLP_REDIS_BACKEND_DB_QUERY": str(self._clp_config.redis.query_backend_database), } - # Connection params + + # Connection config env_vars |= { "CLP_REDIS_HOST": _get_ip_from_hostname(self._clp_config.redis.host), "CLP_REDIS_PORT": str(self._clp_config.redis.port), } + # Credentials env_vars |= { "CLP_REDIS_PASS": self._clp_config.redis.password, } + # Paths env_vars |= { "CLP_REDIS_CONF_FILE_HOST": str(conf_file), @@ -254,12 +262,14 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: self._clp_config.results_cache.stream_collection_name ), } - # Connection params + + # Connection config env_vars |= { "CLP_RESULTS_CACHE_DB_NAME": self._clp_config.results_cache.db_name, "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self._clp_config.results_cache.host), "CLP_RESULTS_CACHE_PORT": str(self._clp_config.results_cache.port), } + # Paths env_vars |= { "CLP_RESULTS_CACHE_CONF_FILE_HOST": str(conf_file), @@ -280,7 +290,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging params + # Logging config env_vars |= { "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": ( self._clp_config.compression_scheduler.logging_level @@ -300,7 +310,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging params + # Logging config env_vars |= { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self._clp_config.query_scheduler.logging_level, } @@ -322,12 +332,13 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging params + # Logging config env_vars |= { "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": ( self._clp_config.compression_worker.logging_level ), } + # Resources env_vars |= { "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_workers), @@ -350,10 +361,11 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging params + # Logging config env_vars |= { "CLP_QUERY_WORKER_LOGGING_LEVEL": self._clp_config.query_worker.logging_level, } + # Resources env_vars |= { "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), @@ -376,10 +388,11 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging params + # Logging config env_vars |= { "CLP_REDUCER_LOGGING_LEVEL": self._clp_config.reducer.logging_level, } + # Resources env_vars |= { "CLP_REDUCER_CONCURRENCY": str(num_workers), @@ -496,12 +509,13 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: env_vars = EnvVarsDict() - # Connection params + # Connection config env_vars |= { "CLP_WEBUI_HOST": _get_ip_from_hostname(self._clp_config.webui.host), "CLP_WEBUI_PORT": str(self._clp_config.webui.port), } - # Security params + + # Security config env_vars |= { "CLP_WEBUI_RATE_LIMIT": str(self._clp_config.webui.rate_limit), } @@ -532,7 +546,7 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging params + # Logging config env_vars |= { "CLP_GARBAGE_COLLECTOR_LOGGING_LEVEL": self._clp_config.garbage_collector.logging_level } From 2bc2d2b32ab84c1c5336a14836d19a4609d6a70a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 22 Oct 2025 10:06:51 -0400 Subject: [PATCH 226/238] Correct AWS profile mount for webui - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/deployment/package/docker-compose.base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 937ea73c09..2ffba1fda3 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -303,7 +303,7 @@ services: published: "${CLP_WEBUI_PORT:-4000}" target: 4000 volumes: - - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" - type: "bind" source: "./var/www/webui/client/settings.json" From f0fa5ae2443d0606d7482ef0acfb33c4d74295c5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 22 Oct 2025 09:59:32 -0400 Subject: [PATCH 227/238] Bump stop grace periods for various services. --- tools/deployment/package/docker-compose.base.yaml | 4 ++-- tools/deployment/package/docker-compose.yaml | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 2ffba1fda3..ddb921ecef 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -5,7 +5,7 @@ x-service-defaults: &service_defaults image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" logging: driver: "local" - stop_grace_period: "3s" + stop_grace_period: "60s" user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:1000}" # Common healthcheck defaults. @@ -248,7 +248,6 @@ services: compression-worker: <<: *service_defaults hostname: "compression_worker" - stop_grace_period: "60s" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}\ :${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" @@ -334,6 +333,7 @@ services: garbage-collector: <<: *service_defaults hostname: "garbage_collector" + stop_grace_period: "10s" deploy: # Value must be either 0 or 1. Set to 0 to disable the garbage collector. replicas: "${CLP_GARBAGE_COLLECTOR_ENABLED:-1}" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index f81ea27280..df8df0d071 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -8,7 +8,7 @@ x-service-defaults: &service_defaults image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" logging: driver: "local" - stop_grace_period: "3s" + stop_grace_period: "60s" user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:1000}" x-healthcheck-defaults: &healthcheck_defaults interval: "30s" @@ -31,6 +31,7 @@ services: query-scheduler: <<: *service_defaults hostname: "query_scheduler" + stop_grace_period: "10s" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}\ :${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" @@ -103,6 +104,7 @@ services: reducer: <<: *service_defaults hostname: "reducer" + stop_grace_period: "10s" environment: CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_REDUCER_LOGGING_LEVEL:-INFO}" From 26c3e32d7c4db01eeaad90eb2fa3ce56775a94f2 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 22 Oct 2025 20:02:38 -0400 Subject: [PATCH 228/238] Rewrite design doc. --- docs/src/dev-docs/design-docker-compose.md | 216 ++++++++++++--------- 1 file changed, 128 insertions(+), 88 deletions(-) mode change 100644 => 100755 docs/src/dev-docs/design-docker-compose.md diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md old mode 100644 new mode 100755 index 9bec8bb49b..6f7cb184b5 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -1,83 +1,38 @@ -# Docker Compose design +# Deployment orchestration -This document explains the technical details of CLP's Docker Compose implementation. - -## Overview - -The Docker Compose implementation follows a controller architecture with a `BaseController` abstract -class and a `DockerComposeController` implementation. +The CLP package is composed of several components that are currently designed to be deployed in a +set of containers that are orchestrated using a framework like [Docker Compose][docker-compose]. +This document explains the architecture of that orchestration and any associated nuances. ## Architecture -### Controller Pattern - -The orchestration uses a controller pattern: - -* `BaseController` (abstract): Defines the interface for provisioning and managing CLP components. -* `DockerComposeController`: Implements Docker Compose-specific logic. - -## Initialization - -The controller performs these initialization steps: - -1. **Provisioning**: Provisions all components and generates component specific configuration - variables. -2. **Configuration Transformation**: The `transform_for_container()` method in `CLPConfig` adapts - configurations for containerized environments -3. **Environment Generation**: Creates a `.env` file with necessary Docker Compose variables - -### Configuration Transformation - -The `transform_for_container()` method in the `CLPConfig` class and related component classes -adapts the configuration for containerized environments by: - -1. Converting host paths to container paths -2. Updating service hostnames to match Docker Compose service names -3. Setting appropriate ports for container communication - -### Environment Variables - -The controller generates a comprehensive set of environment variables that are written to a `.env` -file, including: - -* Component-specific settings (ports, logging levels, concurrency) -* Credentials for database, queue, and Redis services -* Paths for data, logs, archives, and streams -* AWS credentials when needed - -## Deployment Process - -The `start-clp.sh` script executes the `start_clp.py` Python script to orchestrate the deployment. - -### Deployment Types - -CLP supports two deployment types determined by the `package.query_engine` configuration setting: - -1. **BASE**: For deployments using [Presto][presto-integration] as the query engine. Uses only - `docker-compose.base.yaml`. -2. **FULL**: For deployments using CLP's native query engine. Uses both compose files. - -## Docker Compose Files - -The Docker Compose setup uses two files: - -* `docker-compose.base.yaml`: Defines base services for all deployment types, excluding Celery - scheduler and worker components to allow separate Presto [integration][presto-integration]. -* `docker-compose.yaml`: Extends the base file with additional services for complete deployments - -Each file defines services with: - -* Service dependencies via `depends_on` -* Health checks for critical services -* Volume binding mounts for persistent data -* Network configuration -* User permissions +[Figure 1](#figure-1) shows the components (_services_ in orchestrator terminology) in the CLP +package as well as their dependencies. The CLP package consists of several long-running services +(e.g., `database`) and some one-time initialization jobs (e.g., `db-table-creator`). Some of the +long-running services depend on the successful completion of the one-time jobs (e.g., `webui` +depends on `results-cache-indices-creator`), while others depend on the health of other long-running +services (e.g., `compression-scheduler` depends on `queue`). -## Service architecture +[Table 1](#table-1) below lists the services their functions, while [Table 2](#table-2) lists the +one-time initialization jobs and their functions. -The Docker Compose setup includes the following services: +(figure-1)= +::::{card} :::{mermaid} +%%{ + init: { + "theme": "base", + "themeVariables": { + "primaryColor": "#0066cc", + "primaryTextColor": "#fff", + "primaryBorderColor": "transparent", + "lineColor": "#007fff", + "secondaryColor": "#007fff", + "tertiaryColor": "#fff" + } + } +}%% graph LR %% Services database["database (MySQL)"] @@ -119,7 +74,7 @@ graph LR results_cache end - subgraph DB Migration Jobs + subgraph Initialization jobs db_table_creator results_cache_indices_creator end @@ -141,12 +96,14 @@ graph LR end ::: -### Services overview ++++ +**Figure 1**: Orchestration architecture of the services in the CLP package. +:::: -The CLP package is composed of several service components. The tables below list the services and -their functions. +(table-1)= +::::{card} -:::{table} Services +:::{table} :align: left | Service | Description | @@ -161,39 +118,122 @@ their functions. | query_worker | Worker processes for search/aggregation jobs | | reducer | Reducers for performing the final stages of aggregation jobs | | webui | Web server for the UI | -| garbage_collector | Background process for retention control | +| garbage_collector | Process to manage data retention | ::: -### One-time initialization jobs ++++ +**Table 1**: Long-running services in the CLP package. +:::: -We also set up short-lived run-once "services" to initialize some services listed above. +(table-2)= +::::{card} -:::{table} Initialization jobs +:::{table} :align: left -| Job | Description | -|-------------------------------|---------------------------------------------------------| -| db-table-creator | Initializes database tables | -| results-cache-indices-creator | Initializes single-node replica set and sets up indices | +| Job | Description | +|-------------------------------|-------------------------------------------------------| +| db-table-creator | Creates and initializes database tables | +| results-cache-indices-creator | Creates a single-node replica set and sets up indices | ::: -## Troubleshooting ++++ +**Table 2**: One-time initialization jobs in the CLP package. +:::: + +## Code structure + +The orchestration code is split up into: + +* `BaseController` that defines: + * common logic for preparing the environment variables, configuration files, and directories + necessary for each service. + * abstract methods that orchestrator-specific derived classes must implement in order to + orchestrate a deployment. +* `Controller` that implements (and/or overrides) any of the methods in + `BaseController` (`` is a placeholder for the specific orchestrator for which the + class is being implemented). + +## Docker Compose orchestration + +This section explains how we use Docker Compose to orchestrate the CLP package and is broken into +the following subsections: + +* [Setting up the Docker Compose project's environment](#setting-up-the-environment) +* [Starting and stoping the Docker Compose project](#starting-and-stopping-the-project) +* [Deployment types](#deployment-types) +* [Implementation details](#implementation-details) +* [Troubleshooting](#troubleshooting) + +### Setting up the environment + +Several services require configuration values to be passed in through the CLP package's config file, +environment variables, and/or command line arguments. Since the services are running in containers, +some of these configuration values need to be modified for the orchestration environment. +Specifically: + +1. Paths on the host must be converted to appropriate paths in the container. +2. Component hostnames must be converted to service names. +3. Component ports must be converted to the component's default ports. + * This is necessary so that in the Docker Compose project file, we can network services together + using the default port rather than a variable for the configured port. + +To achieve this, before starting the deployment, `DockerComposeController.start` generates: + +* a CLP configuration file (`/var/log/.clp-config.yml` on the host) specific to the + Docker Compose project environment. +* an environment variable file (`/.env`) for any other configuration values. +* any necessary directories (e.g., data output directories). + +The Docker Compose project then passes those environment variables to the relevant services, either +as environment variables or command line arguments, as necessary. + +### Starting and stopping the project + +To start and stop the project, `DockerComposeController` simply invokes `docker compose up` or +`docker compose down` as appropriate. However, to allow multiple CLP packages to be run on the same +host, we explicitly specify a project name for the project, where the name is based on the package's +instance ID. + +### Deployment Types + +CLP supports two deployment types determined by the `package.query_engine` configuration setting. + +1. **BASE**: For deployments using [Presto][presto-integration] as the query engine. This deployment + only uses `docker-compose.base.yaml`. +2. **FULL**: For deployments using one of CLP's native query engines. This uses both + `docker-compose.base.yaml` and `docker-compose.yaml`. + +### Implementation details + +One notable implementation detail is in how we handle mounts that are only necessary under certain +configurations. For instance, the input logs mount is only necessary when the `logs_input.type` is +`fs`. If `logs_input.type` is `s3`, we shouldn't mount some random directory from the user's +host filesystem into the container. However, Docker doesn't provide a mechanism to perform +conditional mounts. Instead, we use Docker's variable interpolation to conditionally mount an empty +tmpfs mount into the container. This strategy is used wherever we need a conditional mount. + +### Troubleshooting If you encounter issues with the Docker Compose deployment: 1. Check service status: + ```bash - docker compose ps + docker compose --project-name clp-package- ps ``` 2. View service logs: + ```bash - docker compose logs + docker compose --project-name clp-package- logs ``` 3. Validate configuration: + ```bash docker compose config ``` +[docker-compose]: https://docs.docker.com/compose/ [presto-integration]: ../user-docs/guides-using-presto.md From 164e27e795a552f022c8ebbf184e0e2cdd9ba946 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 22 Oct 2025 20:03:54 -0400 Subject: [PATCH 229/238] Rename design document file. --- ...ign-docker-compose.md => design-deployment-orchestration.md} | 0 docs/src/dev-docs/index.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/src/dev-docs/{design-docker-compose.md => design-deployment-orchestration.md} (100%) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-deployment-orchestration.md similarity index 100% rename from docs/src/dev-docs/design-docker-compose.md rename to docs/src/dev-docs/design-deployment-orchestration.md diff --git a/docs/src/dev-docs/index.md b/docs/src/dev-docs/index.md index c3bf5e0241..c9146763b1 100644 --- a/docs/src/dev-docs/index.md +++ b/docs/src/dev-docs/index.md @@ -95,7 +95,7 @@ tooling-gh-workflows :hidden: design-project-structure -design-docker-compose +design-deployment-orchestration design-kv-ir-streams/index design-metadata-db From 7759c5b9111f48a5b7946c28c8bbbab8f169e470 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 22 Oct 2025 20:05:14 -0400 Subject: [PATCH 230/238] Lint. --- docs/src/dev-docs/design-deployment-orchestration.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/src/dev-docs/design-deployment-orchestration.md b/docs/src/dev-docs/design-deployment-orchestration.md index 6f7cb184b5..a707c45548 100755 --- a/docs/src/dev-docs/design-deployment-orchestration.md +++ b/docs/src/dev-docs/design-deployment-orchestration.md @@ -6,7 +6,7 @@ This document explains the architecture of that orchestration and any associated ## Architecture -[Figure 1](#figure-1) shows the components (_services_ in orchestrator terminology) in the CLP +[Figure 1](#figure-1) shows the components (*services* in orchestrator terminology) in the CLP package as well as their dependencies. The CLP package consists of several long-running services (e.g., `database`) and some one-time initialization jobs (e.g., `db-table-creator`). Some of the long-running services depend on the successful completion of the one-time jobs (e.g., `webui` @@ -119,6 +119,7 @@ graph LR | reducer | Reducers for performing the final stages of aggregation jobs | | webui | Web server for the UI | | garbage_collector | Process to manage data retention | + ::: +++ @@ -135,6 +136,7 @@ graph LR |-------------------------------|-------------------------------------------------------| | db-table-creator | Creates and initializes database tables | | results-cache-indices-creator | Creates a single-node replica set and sets up indices | + ::: +++ From 0fdf31ce5bac1511fdc5a828d5bca16fb9c216fe Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 22 Oct 2025 20:05:28 -0400 Subject: [PATCH 231/238] Split notes. --- docs/src/dev-docs/building-package.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) mode change 100644 => 100755 docs/src/dev-docs/building-package.md diff --git a/docs/src/dev-docs/building-package.md b/docs/src/dev-docs/building-package.md old mode 100644 new mode 100755 index 0e41f3b75a..c6ae1492bf --- a/docs/src/dev-docs/building-package.md +++ b/docs/src/dev-docs/building-package.md @@ -59,7 +59,9 @@ The build will be in `build/clp-package` and defaults to using the storage engin :::{note} The `task` command runs `task docker-images:package` under the hood. In addition to the build, a Docker image named `clp-package:dev--` will also be created. +::: +:::{note} The package includes a `docker-compose.yaml` file that can be used to deploy CLP using Docker Compose. If you want to manually deploy with Docker Compose instead of using the package scripts, see the [Docker Compose design][docker-compose-design] for more information. @@ -85,7 +87,7 @@ task clean ``` [Docker]: https://docs.docker.com/engine/install/ -[docker-compose-design]: ../dev-docs/design-docker-compose.md +[docker-compose-design]: design-deployment-orchestration.md [Task]: https://taskfile.dev/ [uv]: https://docs.astral.sh/uv/ [y-scope/clp#1352]: https://github.com/y-scope/clp/issues/1352 From 7447fef68caf77288c44fc4477691ba6a2b8354b Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 22 Oct 2025 23:33:56 -0400 Subject: [PATCH 232/238] Tell readers how to find the instance ID. --- docs/src/dev-docs/design-deployment-orchestration.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/src/dev-docs/design-deployment-orchestration.md b/docs/src/dev-docs/design-deployment-orchestration.md index a707c45548..142346009f 100755 --- a/docs/src/dev-docs/design-deployment-orchestration.md +++ b/docs/src/dev-docs/design-deployment-orchestration.md @@ -217,7 +217,9 @@ tmpfs mount into the container. This strategy is used wherever we need a conditi ### Troubleshooting -If you encounter issues with the Docker Compose deployment: +If you encounter issues with the Docker Compose deployment, first determine the instance ID for your +deployment by checking the content of `/var/log/instance-id`. Then run one of the +commands below as necessary. 1. Check service status: From 52272920b2a2407d23285ee7cc1cf5f24bc04728 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 22 Oct 2025 23:35:31 -0400 Subject: [PATCH 233/238] Rename Databases to Shared data stores. --- docs/src/dev-docs/design-deployment-orchestration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/dev-docs/design-deployment-orchestration.md b/docs/src/dev-docs/design-deployment-orchestration.md index 142346009f..79a5770d38 100755 --- a/docs/src/dev-docs/design-deployment-orchestration.md +++ b/docs/src/dev-docs/design-deployment-orchestration.md @@ -67,7 +67,7 @@ graph LR db_table_creator -->|completed_successfully| garbage_collector results_cache_indices_creator -->|completed_successfully| garbage_collector - subgraph Databases + subgraph Shared data stores database queue redis From ed25fc32cbb31a30dbb497d6f14f51b87869cba2 Mon Sep 17 00:00:00 2001 From: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Thu, 23 Oct 2025 09:07:56 -0400 Subject: [PATCH 234/238] Update docs/src/dev-docs/design-deployment-orchestration.md Co-authored-by: Junhao Liao --- docs/src/dev-docs/design-deployment-orchestration.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/src/dev-docs/design-deployment-orchestration.md b/docs/src/dev-docs/design-deployment-orchestration.md index 79a5770d38..c3e4cfb0b6 100755 --- a/docs/src/dev-docs/design-deployment-orchestration.md +++ b/docs/src/dev-docs/design-deployment-orchestration.md @@ -175,10 +175,8 @@ some of these configuration values need to be modified for the orchestration env Specifically: 1. Paths on the host must be converted to appropriate paths in the container. -2. Component hostnames must be converted to service names. -3. Component ports must be converted to the component's default ports. - * This is necessary so that in the Docker Compose project file, we can network services together - using the default port rather than a variable for the configured port. +2. Component hostnames must be converted to service names, and component ports must be converted to the component's default ports. + * This ensures that in the Docker Compose configuration, services can communicate over fixed, predictable hostnames and ports rather than relying on configurable variables. To achieve this, before starting the deployment, `DockerComposeController.start` generates: From 4926450ec990d8b2e921cc71ea70c3d95f76d050 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Thu, 23 Oct 2025 09:16:57 -0400 Subject: [PATCH 235/238] building-package.md: Minor fixes. --- docs/src/dev-docs/building-package.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/src/dev-docs/building-package.md b/docs/src/dev-docs/building-package.md index b6f1d61a26..4149366a12 100755 --- a/docs/src/dev-docs/building-package.md +++ b/docs/src/dev-docs/building-package.md @@ -62,9 +62,10 @@ Docker image named `clp-package:dev--` will also be created. ::: :::{note} -The package includes a `docker-compose.yaml` file that can be used to deploy CLP using Docker -Compose. If you want to manually deploy with Docker Compose instead of using the package scripts, -see the [Docker Compose design][docker-compose-design] for more information. +The package includes a `docker-compose.yaml` file that can be used to deploy CLP using [Docker +Compose][docker-compose]. If you want to manually deploy with Docker Compose instead of using the +package scripts, see the [Deployment orchestration][design-deployment-orchestration] design doc for +more information. ::: To build a releasable tar of either flavour, run: @@ -87,7 +88,8 @@ task clean ``` [Docker]: https://docs.docker.com/engine/install/ -[docker-compose-design]: design-deployment-orchestration.md +[docker-compose]: https://docs.docker.com/compose/ +[design-deployment-orchestration]: design-deployment-orchestration.md [Task]: https://taskfile.dev/ [uv]: https://docs.astral.sh/uv/ [y-scope/clp#1352]: https://github.com/y-scope/clp/issues/1352 From 127cb79372413aaeda3cf09f154744c816a3fffc Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Thu, 23 Oct 2025 09:12:08 -0400 Subject: [PATCH 236/238] Remove other details from multi-node doc until we add multi-node support back in. --- docs/src/user-docs/guides-multi-node.md | 45 ++----------------------- 1 file changed, 3 insertions(+), 42 deletions(-) diff --git a/docs/src/user-docs/guides-multi-node.md b/docs/src/user-docs/guides-multi-node.md index 52f206f7af..697fbf396e 100644 --- a/docs/src/user-docs/guides-multi-node.md +++ b/docs/src/user-docs/guides-multi-node.md @@ -3,47 +3,8 @@ A multi-node deployment allows you to run CLP across a distributed set of hosts. :::{warning} -CLP now uses Docker Compose for orchestration and support for multi-node deployments has been -removed temporarily. Please contact us if you need immediate support for multi-node deployments, or -stay tuned for future updates on Kubernetes Helm support. +CLP now uses [Docker Compose][docker-compose] for orchestration and support for multi-node +deployments has been removed temporarily. ::: -## Setting up SeaweedFS - -The instructions below are for running a simple SeaweedFS cluster on a set of hosts. For other use -cases, see the [SeaweedFS docs][seaweedfs-docs]. - -1. Install [SeaweedFS][seaweedfs-install-docs]. -2. Start the master and a filer on one of the hosts: - - ```bash - weed master -port 9333 - weed filer -port 8888 -master "localhost:9333" - ``` - -3. Start one or more volume servers on one or more hosts. - - {style=lower-alpha} - 1. Create a directory where you want SeaweedFS to store data. - 2. Start the volume server: - - ```bash - weed volume -mserver ":9333" -dir -max 0 - ``` - - * `` is the hostname/IP of the master host. - * `` is the directory where you want SeaweedFS to store data. -4. Start a FUSE mount on every host that you want to run a CLP worker: - - ``` - weed mount -filer ":8888" -dir - ``` - - * `` is the hostname/IP of the master host. - * `` is the path where you want the mount to be. - -[Docker]: https://docs.docker.com/engine/install/ -[docker-non-root]: https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user -[SeaweedFS]: https://github.com/seaweedfs/seaweedfs -[seaweedfs-docs]: https://github.com/seaweedfs/seaweedfs/blob/master/README.md -[seaweedfs-install-docs]: https://github.com/seaweedfs/seaweedfs?tab=readme-ov-file#quick-start-with-single-binary +[docker-compose]: https://docs.docker.com/compose/ From 649276f25af79883068b8c7ed9b7ae05c85f0434 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Thu, 23 Oct 2025 10:45:18 -0400 Subject: [PATCH 237/238] Add comment to explain the `+/tmp` hack. --- .../deployment/package/docker-compose.base.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index ddb921ecef..d691cac474 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -264,6 +264,23 @@ services: - *volume_clp_logs - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" + + # NOTE: Only one of `CLP_ARCHIVE_OUTPUT_DIR_HOST` and `CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST` are + # set at a time, but since `./var/data` on the host is mounted into the container and both + # `CLP_ARCHIVE_OUTPUT_DIR_HOST` and `CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST` default to + # directories under `./var/data`, we need to use a hack to avoid having Docker create the + # unset directory on the host (as root). + # + # For example, let's say we use the following as the mount for staged archives: + # "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/staged-archives". If + # `CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST` is unset, Docker will create + # `/var/data/staged-archives` in the container, but it will also create + # `./var/data/staged-archives` on the host as root; this is because `/var/data` in the + # container is bind mounted to `./var/data` on the host. + # + # The hack to avoid this is if one of `CLP_ARCHIVE_OUTPUT_DIR_HOST` or + # `CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST` is unset, we set the target for the corresponding mount + # to a path that's not under `/var/data` in the container. - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ :${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ From 7139740dda6827b7cad1b8bab81c5f8e8a9a14a5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 23 Oct 2025 10:56:53 -0400 Subject: [PATCH 238/238] docs(design-deployment): Rename "Shared data stores" subgraph to "Databases" --- docs/src/dev-docs/design-deployment-orchestration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/dev-docs/design-deployment-orchestration.md b/docs/src/dev-docs/design-deployment-orchestration.md index c3e4cfb0b6..f7d1423dba 100755 --- a/docs/src/dev-docs/design-deployment-orchestration.md +++ b/docs/src/dev-docs/design-deployment-orchestration.md @@ -67,7 +67,7 @@ graph LR db_table_creator -->|completed_successfully| garbage_collector results_cache_indices_creator -->|completed_successfully| garbage_collector - subgraph Shared data stores + subgraph Databases database queue redis