diff --git a/bin/colcon b/bin/colcon index f543142c9..81228c57a 100755 --- a/bin/colcon +++ b/bin/colcon @@ -105,6 +105,7 @@ custom_extension_points.update({ # there is no point in registering the extension_point extensions here # since they can't be queried through pkg_resources without installing }, + 'colcon_core.output_style': {}, 'colcon_core.package_augmentation': { 'python': PythonPackageAugmentation, }, diff --git a/colcon_core/__init__.py b/colcon_core/__init__.py index d946a00e3..800b50d7a 100644 --- a/colcon_core/__init__.py +++ b/colcon_core/__init__.py @@ -1,4 +1,4 @@ # Copyright 2016-2020 Dirk Thomas # Licensed under the Apache License, Version 2.0 -__version__ = '0.17.0' +__version__ = '0.19.0' diff --git a/colcon_core/command.py b/colcon_core/command.py index 9c94118c6..e21fc62f6 100644 --- a/colcon_core/command.py +++ b/colcon_core/command.py @@ -61,6 +61,8 @@ from colcon_core.logging import colcon_logger # noqa: E402 from colcon_core.logging import get_numeric_log_level # noqa: E402 from colcon_core.logging import set_logger_level_from_env # noqa: E402 +from colcon_core.output_style import add_output_style_arguments # noqa: E402 +from colcon_core.output_style import apply_output_style # noqa: E402 from colcon_core.plugin_system import get_first_line_doc # noqa: E402 from colcon_core.verb import get_verb_extensions # noqa: E402 @@ -93,7 +95,8 @@ def register_command_exit_handler(handler): def main( *, command_name='colcon', argv=None, verb_group_name=None, - environment_variable_group_name=None, + environment_variable_group_name=None, default_verb=None, + default_log_base='log', ): """ Execute the main logic of the command. @@ -114,13 +117,21 @@ def main( :param str command_name: The name of the command invoked :param list argv: The list of arguments + :param str verb_group_name: The extension point group name for verbs + :param str environment_variable_group_name: The extension point group name + for environment variables + :param Type default_verb: The verb class type to invoke if no explicit + verb was provided on the command line + :param str default_log_base: The default logging base path if the command + line argument isn't specified and the environment variable is not set :returns: The return code """ try: return _main( command_name=command_name, argv=argv, verb_group_name=verb_group_name, - environment_variable_group_name=environment_variable_group_name) + environment_variable_group_name=environment_variable_group_name, + default_verb=default_verb, default_log_base=default_log_base) except KeyboardInterrupt: return signal.SIGINT finally: @@ -132,6 +143,7 @@ def main( def _main( *, command_name, argv, verb_group_name, environment_variable_group_name, + default_verb, default_log_base, ): # default log level, for searchability: COLCON_LOG_LEVEL colcon_logger.setLevel(logging.WARNING) @@ -151,6 +163,13 @@ def _main( parser = create_parser(environment_variable_group_name) + if default_verb is not None: + default_verb_instance = default_verb() + parser.set_defaults( + verb_parser=parser, verb_extension=default_verb_instance, + main=default_verb_instance.main) + add_parser_arguments(parser, default_verb_instance) + verb_extensions = get_verb_extensions(group_name=verb_group_name) # add subparsers for all verb extensions but without arguments for now @@ -163,7 +182,7 @@ def _main( known_args, _ = parser.parse_known_args(args=argv) # add the arguments for the requested verb - if known_args.verb_name: + if known_args.verb_name is not None: add_parser_arguments(known_args.verb_parser, known_args.verb_extension) args = parser.parse_args(args=argv) @@ -175,18 +194,24 @@ def _main( colcon_logger.debug(f'Parsed command line arguments: {args}') - # error: no verb provided - if args.verb_name is None: + apply_output_style(args) + + # verify that one of the verbs set the 'main' attribute to be invoked later + if getattr(args, 'main', None) is None: print(parser.format_usage()) return 'Error: No verb provided' # set default locations for log files, for searchability: COLCON_LOG_PATH now = datetime.datetime.now() now_str = str(now)[:-7].replace(' ', '_').replace(':', '-') + if args.verb_name is None: + subdirectory = now_str + else: + subdirectory = f'{args.verb_name}_{now_str}' set_default_log_path( base_path=args.log_base, env_var=f'{command_name}_LOG_PATH'.upper(), - subdirectory=f'{args.verb_name}_{now_str}') + subdirectory=subdirectory, default=default_log_base) # add a file handler writing all levels if logging isn't disabled log_path = get_log_path() @@ -240,9 +265,13 @@ def _parse_optional(self, arg_string): # the option. As of that PR (which is in Python 3.13, and # backported to Python 3.12), it returns a 4-tuple. Check for # either here. + # A similar regression later occurred when _parse_optional() + # started returning a list of tuples, brought in by + # https://github.com/python/cpython/pull/124631 . if result in ( (None, arg_string, None), (None, arg_string, None, None), + [(None, arg_string, None, None)], ): # in the case there the arg is classified as an unknown 'O' # override that and classify it as an 'A' @@ -264,6 +293,7 @@ def _parse_optional(self, arg_string): parser = decorate_argument_parser(parser) add_log_level_argument(parser) + add_output_style_arguments(parser) return parser @@ -428,7 +458,9 @@ def create_subparser(parser, cmd_name, verb_extensions, *, attribute): title=f'{cmd_name} verbs', description='\n'.join(verbs) or None, dest=attribute, - help=f'call `{cmd_name} VERB -h` for specific help' if verbs else None, + help=( + f'call `{cmd_name} VERB -h` for specific help' if + verbs else argparse.SUPPRESS), ) return subparser diff --git a/colcon_core/event_handler/console_start_end.py b/colcon_core/event_handler/console_start_end.py index 990dc926d..c34289227 100644 --- a/colcon_core/event_handler/console_start_end.py +++ b/colcon_core/event_handler/console_start_end.py @@ -4,11 +4,14 @@ import sys import time +import colorama + from colcon_core.event.job import JobEnded from colcon_core.event.job import JobStarted from colcon_core.event.test import TestFailure from colcon_core.event_handler import EventHandlerExtensionPoint from colcon_core.event_handler import format_duration +from colcon_core.output_style import Style from colcon_core.plugin_system import satisfies_version from colcon_core.subprocess import SIGINT_RESULT @@ -25,6 +28,7 @@ class ConsoleStartEndEventHandler(EventHandlerExtensionPoint): def __init__(self): # noqa: D107 super().__init__() + colorama.init() satisfies_version( EventHandlerExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') self._start_times = {} @@ -34,31 +38,47 @@ def __call__(self, event): # noqa: D102 data = event[0] if isinstance(data, JobStarted): + job_id = Style.PackageOrJobName(data.identifier) self._start_times[data.identifier] = time.monotonic() - print(f'Starting >>> {data.identifier}', flush=True) + msg = 'Starting ' + colorama.Fore.GREEN + \ + colorama.Style.BRIGHT + Style.Pictogram('>>>') + colorama.Fore.CYAN + \ + f' {job_id}' + colorama.Style.RESET_ALL + print(Style.SectionStart(msg), flush=True) elif isinstance(data, TestFailure): job = event[1] self._with_test_failures.add(job) elif isinstance(data, JobEnded): + job_id = Style.PackageOrJobName(data.identifier) duration = \ time.monotonic() - self._start_times[data.identifier] - duration_string = format_duration(duration) + duration_string = Style.Measurement(format_duration(duration)) if not data.rc: - msg = f'Finished <<< {data.identifier} [{duration_string}]' + msg = colorama.Style.BRIGHT + colorama.Fore.BLACK + \ + 'Finished ' + colorama.Fore.GREEN + Style.Pictogram('<<<') + \ + colorama.Style.RESET_ALL + colorama.Fore.CYAN + \ + f' {job_id}' + colorama.Fore.RESET + \ + ' [' + colorama.Fore.YELLOW + \ + f'{duration_string}' + colorama.Fore.RESET + ']' job = event[1] if job in self._with_test_failures: - msg += '\t[ with test failures ]' + msg += Style.Warning('\t[ with test failures ]') writable = sys.stdout elif data.rc == SIGINT_RESULT: - msg = f'Aborted <<< {data.identifier} [{duration_string}]' + msg = colorama.Style.BRIGHT + colorama.Fore.RED + \ + Style.Warning('Aborted') + ' ' + colorama.Style.NORMAL + \ + Style.Pictogram('<<<') + colorama.Fore.CYAN + \ + f' {job_id} [{duration_string}]' + colorama.Fore.RESET writable = sys.stdout - else: - msg = f'Failed <<< {data.identifier} ' \ - f'[{duration_string}, exited with code {data.rc}]' + msg = Style.Critical('Failed') + ' ' + \ + Style.Pictogram('<<<') + colorama.Fore.CYAN + f' {job_id} ' + \ + colorama.Fore.CYAN + f' {job_id} [{duration_string}]' + \ + colorama.Fore.RESET + ' [' + colorama.Fore.RED + \ + Style.Error(f'exited with code {data.rc}') + \ + colorama.Fore.RESET + ']' writable = sys.stderr - print(msg, file=writable, flush=True) + print(Style.SectionEnd(msg), file=writable, flush=True) diff --git a/colcon_core/executor/__init__.py b/colcon_core/executor/__init__.py index c0bdaccf9..e3eed87d6 100644 --- a/colcon_core/executor/__init__.py +++ b/colcon_core/executor/__init__.py @@ -141,7 +141,7 @@ class ExecutorExtensionPoint: """ """The version of the executor extension interface.""" - EXTENSION_POINT_VERSION = '1.0' + EXTENSION_POINT_VERSION = '1.1' """The default priority of executor extensions.""" PRIORITY = 100 @@ -191,6 +191,14 @@ def execute( """ raise NotImplementedError() + def put_event_into_queue(self, event): + """ + Post a message event into the event queue. + + :param event: The event + """ + self._event_controller.get_queue().put((event, self)) + def _flush(self): if self._event_controller is None: return diff --git a/colcon_core/location.py b/colcon_core/location.py index f65517ccb..8924929f7 100644 --- a/colcon_core/location.py +++ b/colcon_core/location.py @@ -211,10 +211,13 @@ def create_log_path(verb_name): ignore_marker.touch() # create latest symlinks - _create_symlink(path, path.parent / f'latest_{verb_name}') - _create_symlink( - path.parent / f'latest_{verb_name}', - path.parent / 'latest') + if verb_name is None: + _create_symlink(path, path.parent / 'latest') + else: + _create_symlink(path, path.parent / f'latest_{verb_name}') + _create_symlink( + path.parent / f'latest_{verb_name}', + path.parent / 'latest') def _reset_log_path_creation_global(): diff --git a/colcon_core/output_style/__init__.py b/colcon_core/output_style/__init__.py new file mode 100644 index 000000000..dac2dd54c --- /dev/null +++ b/colcon_core/output_style/__init__.py @@ -0,0 +1,193 @@ +# Copyright 2025 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from collections import namedtuple +import os +from types import SimpleNamespace + +from colcon_core.environment_variable import EnvironmentVariable +from colcon_core.plugin_system import get_first_line_doc +from colcon_core.plugin_system import instantiate_extensions +from colcon_core.plugin_system import order_extensions_grouped_by_priority + +"""Environment variable to override the default output style""" +DEFAULT_OUTPUT_STYLE_ENVIRONMENT_VARIABLE = EnvironmentVariable( + 'COLCON_DEFAULT_OUTPUT_STYLE', 'Select the default output style extension') + + +class Stylizer(namedtuple('Stylizer', ('start', 'end'))): + """A text style modifier.""" + + __slots__ = () + + def __new__(cls, start=None, end=None): # noqa: D102 + if start is None: + start = '' + if end is None: + end = '' + return super(Stylizer, cls).__new__(cls, start, end) + + def __add__(self, other): + """Combine two modifiers into a single modifier.""" + if not isinstance(other, Stylizer): + raise TypeError() + return Stylizer( + self.start + other.start, + other.end + self.end) + + def __call__(self, text): + """ + Apply style modification to the given text. + + :param text: Text to be modified + :returns: Modified text + """ + return self.start + text + self.end + + +Stylizer.Default = Stylizer() + + +class StyleCollection(SimpleNamespace): + """Collection of Stylizers to use when styling console output.""" + + Default = Stylizer.Default + + def __getattr__(self, name): # noqa: D105 + if name.startswith('_'): + return super().__getattr__(name) + return self.Default + + +Style = StyleCollection( + Critical=Stylizer.Default, + Default=Stylizer.Default, + Error=Stylizer.Default, + Measurement=Stylizer.Default, + PackageOrJobName=Stylizer.Default, + Path=Stylizer.Default, + Pictogram=Stylizer.Default, + SectionEnd=Stylizer.Default, + SectionStart=Stylizer.Default, + Strong=Stylizer.Default, + Success=Stylizer.Default, + Warning=Stylizer.Default, + Weak=Stylizer.Default, +) + + +class OutputStyleExtensionPoint: + """The interface for stylizing colcon output.""" + + """The version of the output style extension interface.""" + EXTENSION_POINT_VERSION = '1.0' + + """The default priority of output style extensions.""" + PRIORITY = 100 + + def __init__(self): # noqa: D107 + super().__init__() + + def apply_style(self, style): + """ + Apply output style modifications. + + :param style: The current output style + """ + raise NotImplementedError() + + +def get_output_style_extensions(*, group_name=None): + """ + Get the available output style extensions. + + The extensions are grouped by their priority and each group is ordered by + the entry point name. + + :rtype: OrderedDict + """ + if group_name is None: + group_name = __name__ + extensions = instantiate_extensions(group_name) + return order_extensions_grouped_by_priority(extensions) + + +def add_output_style_arguments(parser, *, extensions=None): + """ + Add the command line arguments for the output style extensions. + + :param parser: The argument parser + :param extensions: The output style extensions to use, if `None` is passed + use the extensions provided by + :function:`get_output_style_extensions` + """ + if extensions is None: + extensions = get_output_style_extensions() + keys = [] + descriptions = '' + default = None + for priority in extensions.keys(): + extensions_same_prio = extensions[priority] + assert len(extensions_same_prio) == 1, \ + 'Output style extensions must have unique priorities' + for key, extension in extensions_same_prio.items(): + keys.append(key) + if default is None and priority >= 100: + default = key + desc = get_first_line_doc(extension) + if not desc: + # show extensions without a description + # to mention the available options + desc = '' + # it requires a custom formatter to maintain the newline + descriptions += f'\n* {key}: {desc}' + + if not keys: + return + + default = os.environ.get( + DEFAULT_OUTPUT_STYLE_ENVIRONMENT_VARIABLE.name, + default) + parser.add_argument( + '--output-style', type=str, choices=keys, default=default, + help='The style extension to use when producing output ' + f'(default: {default}){descriptions}') # noqa: E131 + + +def select_output_style_extension(args, *, extensions=None): + """ + Get the output style extension. + + :param args: The parsed command line arguments + :param extensions: The output style extensions to use, if `None` is passed + use the extensions provided by + :function:`get_output_style_extensions` + + :returns: The output style extension (or None if not available) + """ + if extensions is None: + extensions = get_output_style_extensions() + for priority in extensions.keys(): + extensions_same_prio = extensions[priority] + for key, extension in extensions_same_prio.items(): + if key == args.output_style: + return extension + + +def apply_output_style(args, *, extensions=None): + """ + Apply output style for the appropriate extension if any are available. + + :param args: The parsed command line arguments + :param extensions: The output style extensions to use, if `None` is passed + use the extensions provided by + :function:`get_output_style_extensions` + """ + # TODO: This approach chooses only a single extension. Should it be + # possible to apply styles on top of each other, possibly ones which + # serve different purposes? + # Expressing styles similar to event handlers might be the only way + # to expose that on the command line. + extension = select_output_style_extension(args, extensions=extensions) + if extension is not None: + extension.apply_style(Style) diff --git a/colcon_core/output_style/__main__.py b/colcon_core/output_style/__main__.py new file mode 100644 index 000000000..40c391134 --- /dev/null +++ b/colcon_core/output_style/__main__.py @@ -0,0 +1,22 @@ +# Copyright 2025 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import copy + +from colcon_core.output_style import get_output_style_extensions +from colcon_core.output_style import Style +from colcon_core.output_style import StyleCollection +from colcon_core.output_style import Stylizer + + +blank_style = StyleCollection( + **{k: Stylizer.Default for k in vars(Style).keys()}) + + +extension_groups = get_output_style_extensions() +for extension_group in extension_groups.values(): + for name, extension in extension_group.items(): + style = copy.copy(blank_style) + extension.apply_style(style) + demo = '\n - '.join(v(k) for k, v in vars(style).items()) + print(f'{name}:\n - ' + demo) diff --git a/colcon_core/package_discovery/__init__.py b/colcon_core/package_discovery/__init__.py index c16f3b1d3..9d73b0f4c 100644 --- a/colcon_core/package_discovery/__init__.py +++ b/colcon_core/package_discovery/__init__.py @@ -25,14 +25,14 @@ class PackageDiscoveryExtensionPoint: """ """The version of the discovery extension interface.""" - EXTENSION_POINT_VERSION = '1.0' + EXTENSION_POINT_VERSION = '1.1' """The default priority of discovery extensions.""" PRIORITY = 100 def has_default(self): """ - Check if the extension has a default parameter is none are provided. + Check if the extension has a default parameter if none are provided. The method is intended to be overridden in a subclass. @@ -62,7 +62,9 @@ def has_parameters(self, *, args): This method must be overridden in a subclass. :param args: The parsed command line arguments - :returns: True if `discover()` should be called, False otherwise + :returns: True if `discover()` should be called, False if no + parameters were given, or None if this extension has no + parameters to be specified. :rtype: bool """ raise NotImplementedError() @@ -219,6 +221,7 @@ def expand_dir_wildcards(paths): def _get_extensions_with_parameters( args, discovery_extensions ): + explicitly_specified = False with_parameters = OrderedDict() for extension in discovery_extensions.values(): logger.log( @@ -235,8 +238,11 @@ def _get_extensions_with_parameters( # skip failing extension, continue with next one else: if has_parameter: - with_parameters[extension.PACKAGE_DISCOVERY_NAME] = extension - return with_parameters + explicitly_specified = True + elif has_parameter is not None: + continue + with_parameters[extension.PACKAGE_DISCOVERY_NAME] = extension + return with_parameters if explicitly_specified else OrderedDict() def _discover_packages( diff --git a/colcon_core/package_selection/__init__.py b/colcon_core/package_selection/__init__.py index e76a474b0..37c5df881 100644 --- a/colcon_core/package_selection/__init__.py +++ b/colcon_core/package_selection/__init__.py @@ -71,7 +71,9 @@ def select_packages(self, *, args, decorators): raise NotImplementedError() -def add_arguments(parser): +def add_arguments( + parser, *, discovery_extensions=None, selection_extensions=None, +): """ Add the command line arguments for the package selection extensions. @@ -79,10 +81,16 @@ def add_arguments(parser): the package discovery arguments. :param parser: The argument parser + :param discovery_extensions: The package discovery extensions to use, if + `None` is passed use the extensions provided by + :function:`get_package_discovery_extensions` + :param selection_extensions: The package selection extensions to use, if + `None` is passed use the extensions provided by + :function:`get_package_selection_extensions` """ - add_package_discovery_arguments(parser) + add_package_discovery_arguments(parser, extensions=discovery_extensions) - _add_package_selection_arguments(parser) + _add_package_selection_arguments(parser, extensions=selection_extensions) def get_package_selection_extensions(*, group_name=None): @@ -101,15 +109,16 @@ def get_package_selection_extensions(*, group_name=None): return order_extensions_by_priority(extensions) -def _add_package_selection_arguments(parser): +def _add_package_selection_arguments(parser, *, extensions=None): """ Add the command line arguments for the package selection extensions. :param parser: The argument parser """ - package_selection_extensions = get_package_selection_extensions() + if extensions is None: + extensions = get_package_selection_extensions() group = parser.add_argument_group(title='Package selection arguments') - for extension in package_selection_extensions.values(): + for extension in extensions.values(): try: retval = extension.add_arguments(parser=group) assert retval is None, 'add_arguments() should return None' @@ -125,7 +134,9 @@ def _add_package_selection_arguments(parser): def get_packages( args, *, additional_argument_names=None, - direct_categories=None, recursive_categories=None + direct_categories=None, recursive_categories=None, + discovery_extensions=None, identification_extensions=None, + augmentation_extensions=None, selection_extensions=None, ): """ Get the selected package decorators in topological order. @@ -141,17 +152,35 @@ def get_packages( :param Iterable[str]|Mapping[str, Iterable[str]] recursive_categories: The names of the recursive categories, optionally mapped from the immediate upstream category which included the dependency + :param discovery_extensions: The package discovery extensions to use, if + `None` is passed use the extensions provided by + :function:`get_package_discovery_extensions` + :param identification_extensions: The package identification extensions to + use, if `None` is passed use the extensions provided by + :function:`get_package_identification_extensions` + :param augmentation_extensions: The package augmentation extensions, if + `None` is passed use the extensions provided by + :function:`get_package_augmentation_extensions` + :param selection_extensions: The package selection extensions to use, if + `None` is passed use the extensions provided by + :function:`get_package_selection_extensions` :rtype: list :raises RuntimeError: if the returned set of packages contains duplicates package names """ descriptors = get_package_descriptors( - args, additional_argument_names=additional_argument_names) + args, additional_argument_names=additional_argument_names, + discovery_extensions=discovery_extensions, + identification_extensions=identification_extensions, + augmentation_extensions=augmentation_extensions, + selection_extensions=selection_extensions) decorators = topological_order_packages( descriptors, direct_categories=direct_categories, recursive_categories=recursive_categories) - select_package_decorators(args, decorators) + select_package_decorators( + args, decorators, + selection_extensions=selection_extensions) # check for duplicate package names pkgs = [m.descriptor for m in decorators if m.selected] @@ -169,7 +198,11 @@ def get_packages( return decorators -def get_package_descriptors(args, *, additional_argument_names=None): +def get_package_descriptors( + args, *, additional_argument_names=None, discovery_extensions=None, + identification_extensions=None, augmentation_extensions=None, + selection_extensions=None, +): """ Get the package descriptors. @@ -181,24 +214,44 @@ def get_package_descriptors(args, *, additional_argument_names=None): :param additional_argument_names: A list of additional arguments to consider + :param discovery_extensions: The package discovery extensions to use, if + `None` is passed use the extensions provided by + :function:`get_package_discovery_extensions` + :param identification_extensions: The package identification extensions to + use, if `None` is passed use the extensions provided by + :function:`get_package_identification_extensions` + :param augmentation_extensions: The package augmentation extensions, if + `None` is passed use the extensions provided by + :function:`get_package_augmentation_extensions` + :param selection_extensions: The package selection extensions to use, if + `None` is passed use the extensions provided by + :function:`get_package_selection_extensions` :returns: set of :py:class:`colcon_core.package_descriptor.PackageDescriptor` :rtype: set """ - extensions = get_package_identification_extensions() - descriptors = discover_packages(args, extensions) + if identification_extensions is None: + identification_extensions = get_package_identification_extensions() + descriptors = discover_packages( + args, identification_extensions, + discovery_extensions=discovery_extensions) pkg_names = {d.name for d in descriptors} - _check_package_selection_parameters(args, pkg_names) + _check_package_selection_parameters( + args, pkg_names, selection_extensions=selection_extensions) augment_packages( - descriptors, additional_argument_names=additional_argument_names) + descriptors, additional_argument_names=additional_argument_names, + augmentation_extensions=augmentation_extensions) return descriptors -def _check_package_selection_parameters(args, pkg_names): - package_selection_extensions = get_package_selection_extensions() - for extension in package_selection_extensions.values(): +def _check_package_selection_parameters( + args, pkg_names, *, selection_extensions=None, +): + if selection_extensions is None: + selection_extensions = get_package_selection_extensions() + for extension in selection_extensions.values(): try: retval = extension.check_parameters(args=args, pkg_names=pkg_names) assert retval is None, 'check_parameters() should return None' @@ -211,7 +264,9 @@ def _check_package_selection_parameters(args, pkg_names): # skip failing extension, continue with next one -def select_package_decorators(args, decorators): +def select_package_decorators( + args, decorators, *, selection_extensions=None, +): """ Select the package decorators based on the command line arguments. @@ -219,11 +274,15 @@ def select_package_decorators(args, decorators): :param args: The parsed command line arguments :param list decorators: The package decorators in topological order + :param selection_extensions: The package selection extensions to use, if + `None` is passed use the extensions provided by + :function:`get_package_selection_extensions` """ # filtering must happen after the topological ordering since otherwise # packages in the middle of the dependency graph might be missing - package_selection_extensions = get_package_selection_extensions() - for extension in package_selection_extensions.values(): + if selection_extensions is None: + selection_extensions = get_package_selection_extensions() + for extension in selection_extensions.values(): try: retval = extension.select_packages( args=args, decorators=decorators) diff --git a/colcon_core/python_project/__init__.py b/colcon_core/python_project/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/colcon_core/python_project/spec.py b/colcon_core/python_project/spec.py new file mode 100644 index 000000000..3f4a363be --- /dev/null +++ b/colcon_core/python_project/spec.py @@ -0,0 +1,52 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +try: + # Python 3.11+ + from tomllib import loads as toml_loads +except ImportError: + try: + from tomli import loads as toml_loads + except ImportError: + from toml import loads as toml_loads + + +SPEC_NAME = 'pyproject.toml' + +_DEFAULT_BUILD_SYSTEM = { + 'build-backend': 'setuptools.build_meta:__legacy__', + 'requires': ['setuptools >= 40.8.0', 'wheel'], +} + + +def load_spec(project_path): + """ + Load build system specifications for a Python project. + + :param project_path: Path to the root directory of the project + """ + spec_file = project_path / SPEC_NAME + try: + with spec_file.open('rb') as f: + spec = toml_loads(f.read().decode()) + except FileNotFoundError: + spec = {} + + spec.setdefault('build-system', _DEFAULT_BUILD_SYSTEM) + + return spec + + +def load_and_cache_spec(desc): + """ + Get the cached spec for a package descriptor. + + If the spec has not been loaded yet, load and cache it. + + :param desc: The package descriptor + """ + spec = desc.metadata.get('python_project_spec') + if spec is None: + spec = load_spec(desc.path) + desc.metadata['python_project_spec'] = spec + return spec diff --git a/colcon_core/shell/__init__.py b/colcon_core/shell/__init__.py index 7924b7953..b77e4ede1 100644 --- a/colcon_core/shell/__init__.py +++ b/colcon_core/shell/__init__.py @@ -332,6 +332,40 @@ async def get_command_environment(task_name, build_base, dependencies): 'Could not find a shell extension for the command environment') +async def get_null_separated_environment_variables( + cmd, *, cwd=None, shell=True, +): + """ + Get null-separated environment variables from the output of the command. + + :param args: the sequence of program arguments + :param cwd: the working directory for the subprocess + :param shell: whether to use the shell as the program to execute + :rtype: dict + """ + encoding = locale.getpreferredencoding() + output = await check_output(cmd, cwd=cwd, shell=shell) + env = OrderedDict() + for kvp in output.split(b'\0'): + kvp = kvp.rstrip() + if not kvp: + continue + try: + parts = kvp.decode(encoding).split('=', 1) + except UnicodeDecodeError: + kvp_replaced = kvp.decode(encoding=encoding, errors='replace') + logger.warning( + 'Failed to decode line from the environment using the ' + f"encoding '{encoding}': {kvp_replaced}") + continue + if len(parts) != 2: + # skip lines which don't contain an equal sign + continue + env[parts[0]] = parts[1] + assert len(env) > 0, "The environment shouldn't be empty" + return env + + async def get_environment_variables(cmd, *, cwd=None, shell=True): """ Get the environment variables from the output of the command. diff --git a/colcon_core/shell/sh.py b/colcon_core/shell/sh.py index 640055126..cfab571f4 100644 --- a/colcon_core/shell/sh.py +++ b/colcon_core/shell/sh.py @@ -3,6 +3,7 @@ from pathlib import Path import sys +import warnings from colcon_core import shell from colcon_core.plugin_system import satisfies_version @@ -10,6 +11,7 @@ from colcon_core.prefix_path import get_chained_prefix_path from colcon_core.shell import check_dependency_availability from colcon_core.shell import get_environment_variables +from colcon_core.shell import get_null_separated_environment_variables from colcon_core.shell import logger from colcon_core.shell import ShellExtensionPoint from colcon_core.shell.template import expand_template @@ -150,8 +152,20 @@ async def generate_command_environment( # noqa: D102 hook_path, {'dependencies': dependencies}) - cmd = ['.', str(hook_path), '&&', 'env'] - env = await get_environment_variables(cmd, cwd=str(build_base)) + # Attempt to use null-separated env output, but fall back to the + # best-effort implementation if not supported (i.e. older macOS) + cmd = ['.', str(hook_path), '&&', 'env', '-0'] + try: + env = await get_null_separated_environment_variables( + cmd, cwd=str(build_base)) + except AssertionError: + warnings.warn( + 'This platform does not support null-separated output from ' + "'env', and may not be supported in future releases of " + 'colcon-core. See colcon/colcon-core#684 for details.', + DeprecationWarning) + cmd.pop() + env = await get_environment_variables(cmd, cwd=str(build_base)) # write environment variables to file for debugging env_path = build_base / ('colcon_command_prefix_%s.sh.env' % task_name) diff --git a/colcon_core/shell/template/__init__.py b/colcon_core/shell/template/__init__.py index 842220c89..c04fce3dc 100644 --- a/colcon_core/shell/template/__init__.py +++ b/colcon_core/shell/template/__init__.py @@ -4,10 +4,10 @@ from io import StringIO import os +from colcon_core.generic_decorator import GenericDecorator from colcon_core.logging import colcon_logger try: from em import Interpreter - from em import OVERRIDE_OPT except ImportError as e: try: import em # noqa: F401 @@ -34,13 +34,27 @@ def expand_template(template_path, destination_path, data): """ output = StringIO() try: - # disable OVERRIDE_OPT to avoid saving / restoring stdout - interpreter = CachingInterpreter( - output=output, options={OVERRIDE_OPT: False}) - with template_path.open('r') as h: - content = h.read() - interpreter.string(content, str(template_path), locals=data) - output = output.getvalue() + try: + from em import Configuration + except ImportError: + from em import OVERRIDE_OPT + # disable OVERRIDE_OPT to avoid saving / restoring stdout + interpreter = CachingInterpreter( + output=output, options={OVERRIDE_OPT: False}) + else: + interpreter = CachingInterpreter( + output=output, + config=Configuration( + defaultRoot=str(template_path), + useProxy=False), + dispatcher=False) + try: + with template_path.open('r') as h: + content = h.read() + interpreter.string(content, locals=data) + output = output.getvalue() + finally: + interpreter.shutdown() except Exception as e: # noqa: F841 logger.error( f"{e.__class__.__name__} processing template '{template_path}'") @@ -53,41 +67,47 @@ def expand_template(template_path, destination_path, data): destination_path.unlink() with destination_path.open('w') as h: h.write(output) - finally: - interpreter.shutdown() class BypassStdoutInterpreter(Interpreter): """Interpreter for EmPy which keeps `stdout` unchanged.""" - def installProxy(self): # noqa: D102 N802 + def installProxy(self, *args, **kwargs): # noqa: D102 N802 # avoid replacing stdout with ProxyFile - pass + # in EmPy 3.x, this function performed in-place modification. + # in EmPy 4.x, it is passed the output stream and is expected to + # return the output stream. + return next(iter(args), None) cached_tokens = {} class CachingInterpreter(BypassStdoutInterpreter): - """Interpreter for EmPy which which caches parsed tokens.""" - - def parse(self, scanner, locals=None): # noqa: A002 D102 - global cached_tokens - data = scanner.buffer - # try to use cached tokens - tokens = cached_tokens.get(data) - if tokens is None: - # collect tokens and cache them - tokens = [] - while True: - token = scanner.one() - if token is None: - break - tokens.append(token) - cached_tokens[data] = tokens - - # reimplement the parse method using the (cached) tokens - self.invoke('atParse', scanner=scanner, locals=locals) - for token in tokens: - self.invoke('atToken', token=token) - token.run(self, locals) + """Interpreter for EmPy which caches parsed tokens.""" + + class _CachingScannerDecorator(GenericDecorator): + + def __init__(self, decoree, cache): + super().__init__(decoree, _cache=cache, _idx=0) + + def one(self, *args, **kwargs): + if self._idx < len(self._cache): + token, count = self._cache[self._idx] + self.advance(count) + self.sync() + else: + count = len(self._decoree) + token = self._decoree.one(*args, **kwargs) + count -= len(self._decoree) + self._cache.append((token, count)) + + self._idx += 1 + return token + + def parse(self, scanner, *args, **kwargs): # noqa: A002 D102 + cache = cached_tokens.setdefault(scanner.buffer, []) + return super().parse( + CachingInterpreter._CachingScannerDecorator(scanner, cache), + *args, + **kwargs) diff --git a/colcon_core/task/python/build.py b/colcon_core/task/python/build.py index 5320a939c..30b2ae895 100644 --- a/colcon_core/task/python/build.py +++ b/colcon_core/task/python/build.py @@ -29,6 +29,8 @@ sys.executable, '-W', 'ignore:setup.py install is deprecated', + '-W', + 'ignore:easy_install command is deprecated', ] diff --git a/debian/patches/setup.cfg.patch b/debian/patches/setup.cfg.patch index add781c29..a06666c08 100644 --- a/debian/patches/setup.cfg.patch +++ b/debian/patches/setup.cfg.patch @@ -5,7 +5,7 @@ Author: Dirk Thomas --- setup.cfg 2018-05-27 11:22:33.000000000 -0700 +++ setup.cfg.patched 2018-05-27 11:22:33.000000000 -0700 -@@ -33,9 +33,12 @@ +@@ -33,11 +33,16 @@ importlib-metadata; python_version < "3.8" packaging pytest @@ -19,5 +19,10 @@ Author: Dirk Thomas + # pytest-repeat + # pytest-rerunfailures setuptools>=30.3.0 +- tomli>=1.0.0; python_version < "3.11" ++ # toml is also supported, rely on deb dependencies to select the ++ # appropriate package ++ # tomli>=1.0.0; python_version < "3.11" packages = find: zip_safe = false + diff --git a/setup.cfg b/setup.cfg index 3928d02e7..0177b02bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,9 +27,10 @@ keywords = colcon [options] python_requires = >=3.6 install_requires = + colorama coloredlogs; sys_platform == 'win32' distlib - EmPy<4 + EmPy importlib-metadata; python_version < "3.8" packaging # the pytest dependency and its extensions are provided for convenience @@ -39,6 +40,8 @@ install_requires = pytest-repeat pytest-rerunfailures setuptools>=30.3.0 + # toml is also supported but deprecated + tomli>=1.0.0; python_version < "3.11" packages = find: zip_safe = false @@ -57,7 +60,7 @@ test = pylint pytest pytest-cov - scspell3k>=2.2 + scspell3k>=2.2; python_version < "3.13" [options.packages.find] exclude = @@ -75,6 +78,7 @@ filterwarnings = ignore:The loop argument is deprecated::asyncio ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated::pydocstyle ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated::pyreadline + default:This platform does not support null-separated output:DeprecationWarning:colcon_core.shell.sh junit_suite_name = colcon-core markers = flake8 @@ -93,6 +97,7 @@ colcon_core.environment_variable = extension_blocklist = colcon_core.extension_point:EXTENSION_BLOCKLIST_ENVIRONMENT_VARIABLE home = colcon_core.command:HOME_ENVIRONMENT_VARIABLE log_level = colcon_core.command:LOG_LEVEL_ENVIRONMENT_VARIABLE + output_style = colcon_core.output_style:DEFAULT_OUTPUT_STYLE_ENVIRONMENT_VARIABLE warnings = colcon_core.command:WARNINGS_ENVIRONMENT_VARIABLE colcon_core.event_handler = console_direct = colcon_core.event_handler.console_direct:ConsoleDirectEventHandler @@ -105,6 +110,7 @@ colcon_core.extension_point = colcon_core.environment = colcon_core.environment:EnvironmentExtensionPoint colcon_core.event_handler = colcon_core.event_handler:EventHandlerExtensionPoint colcon_core.executor = colcon_core.executor:ExecutorExtensionPoint + colcon_core.output_style = colcon_core.output_style:OutputStyleExtensionPoint colcon_core.package_augmentation = colcon_core.package_augmentation:PackageAugmentationExtensionPoint colcon_core.package_discovery = colcon_core.package_discovery:PackageDiscoveryExtensionPoint colcon_core.package_identification = colcon_core.package_identification:PackageIdentificationExtensionPoint @@ -116,6 +122,7 @@ colcon_core.extension_point = colcon_core.task.build = colcon_core.task:TaskExtensionPoint colcon_core.task.test = colcon_core.task:TaskExtensionPoint colcon_core.verb = colcon_core.verb:VerbExtensionPoint +colcon_core.output_style = colcon_core.package_augmentation = python = colcon_core.package_augmentation.python:PythonPackageAugmentation colcon_core.package_discovery = diff --git a/stdeb.cfg b/stdeb.cfg index 9eb1227d1..d66507caf 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,8 +1,9 @@ [colcon-core] No-Python2: -Depends3: python3-distlib, python3-empy (<4), python3-packaging, python3-pytest, python3-setuptools, python3 (>= 3.8) | python3-importlib-metadata +Depends3: python3-colorama, python3-distlib, python3-empy, python3-packaging, python3-pytest, python3-setuptools, python3 (>= 3.8) | python3-importlib-metadata, python3 (>= 3.11) | python3-tomli (>= 1) | python3-toml Recommends3: python3-pytest-cov Suggests3: python3-pytest-repeat, python3-pytest-rerunfailures Replaces3: colcon Suite: focal jammy noble bookworm trixie X-Python3-Version: >= 3.6 +Upstream-Version-Suffix: +upstream diff --git a/test/spell_check.words b/test/spell_check.words index 18c2677bf..09aa8e878 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -4,6 +4,7 @@ apache argparse asyncio autouse +backend backported basepath bazqux @@ -85,6 +86,7 @@ prepending proactor purelib pydocstyle +pyproject pytest pytests pythondontwritebytecode @@ -104,6 +106,7 @@ rindex rmtree rstrip rtype +runpy samefile scspell sdist @@ -122,6 +125,8 @@ stacklevel staticmethod stdeb stringify +stylizer +stylizers subparser subparsers subprocesses @@ -136,6 +141,9 @@ testsuite thomas tmpdir todo +toml +tomli +tomllib traceback tryfirst tuples diff --git a/test/test_command.py b/test/test_command.py index c8c6b6e9e..8520ab6e0 100644 --- a/test/test_command.py +++ b/test/test_command.py @@ -39,6 +39,7 @@ def add_arguments(self, *, parser): raise RuntimeError('custom exception') +@patch('colcon_core.output_style.get_output_style_extensions', dict) def test_main(): with ExtensionPointContext( extension1=Extension1, extension2=Extension2, extension3=Extension3 @@ -96,6 +97,29 @@ def test_main_no_verbs_or_env(): assert e.value.code == 0 +def test_main_default_verb(): + with ExtensionPointContext(): + with patch( + 'colcon_core.argument_parser.get_argument_parser_extensions', + return_value={} + ): + with pytest.raises(SystemExit) as e: + main(argv=['--help'], default_verb=Extension1) + assert e.value.code == 0 + + with pytest.raises(SystemExit) as e: + main( + argv=['--log-level', 'invalid'], + default_verb=Extension1) + assert e.value.code == 2 + + with patch.object(Extension1, 'main', return_value=0) as mock_main: + assert not main( + argv=['--log-base', '/dev/null'], + default_verb=Extension1) + mock_main.assert_called_once() + + def test_create_parser(): with ExtensionPointContext(): parser = create_parser('colcon_core.environment_variable') diff --git a/test/test_entry_point.py b/test/test_entry_point.py index bbb797b1d..ba9752c16 100644 --- a/test/test_entry_point.py +++ b/test/test_entry_point.py @@ -2,21 +2,29 @@ # Licensed under the Apache License, Version 2.0 import os +import sys from unittest.mock import Mock from unittest.mock import patch import warnings +import pytest + with warnings.catch_warnings(): warnings.filterwarnings( 'ignore', message='.*entry_point.*deprecated.*', category=UserWarning) - from colcon_core.entry_point import EXTENSION_POINT_GROUP_NAME - from colcon_core.entry_point import get_all_entry_points - from colcon_core.entry_point import get_entry_points - from colcon_core.entry_point import load_entry_point - from colcon_core.entry_point import load_entry_points - -import pytest + try: + from colcon_core.entry_point import EXTENSION_POINT_GROUP_NAME + from colcon_core.entry_point import get_all_entry_points + from colcon_core.entry_point import get_entry_points + from colcon_core.entry_point import load_entry_point + from colcon_core.entry_point import load_entry_points + except ModuleNotFoundError: + if sys.version_info >= (3, 13): + pytest.skip( + 'pkg_resources is no longer available in Python 3.13+', + allow_module_level=True) + raise from .environment_context import EnvironmentContext diff --git a/test/test_executor.py b/test/test_executor.py index 80c2f812c..d7c24ae86 100644 --- a/test/test_executor.py +++ b/test/test_executor.py @@ -123,9 +123,12 @@ def test_interface(): interface = ExecutorExtensionPoint() interface._flush() event_controller = Mock() + event_controller.get_queue.returns = Mock() interface.set_event_controller(event_controller) + interface.put_event_into_queue(object()) interface._flush() assert event_controller.flush.call_count == 1 + assert event_controller.get_queue().put.call_count == 1 class Extension1(ExecutorExtensionPoint): diff --git a/test/test_location.py b/test/test_location.py index 61a8656c0..0141feae5 100644 --- a/test/test_location.py +++ b/test/test_location.py @@ -190,6 +190,17 @@ def test_create_log_path(reset_log_path_creation_global): assert (log_path / subdirectory).exists() assert not (log_path / 'latest').is_symlink() + # check that `latest_verb` is skipped when there is no verb + (log_path / subdirectory).rmdir() + (log_path / 'latest').rmdir() + (log_path / 'latest_verb').unlink() + location._create_log_path_called = False + create_log_path(None) + assert (log_path / subdirectory).exists() + assert (log_path / 'latest').is_symlink() + assert (log_path / 'latest').resolve() == \ + (log_path / subdirectory).resolve() + def test__create_symlink(): # check cases where functions raise exceptions and ensure it is being diff --git a/test/test_output_style.py b/test/test_output_style.py new file mode 100644 index 000000000..dc88c1d3a --- /dev/null +++ b/test/test_output_style.py @@ -0,0 +1,129 @@ +# Copyright 2025 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import argparse +from io import StringIO +import runpy +from types import SimpleNamespace +from unittest.mock import patch + +from colcon_core.output_style import __main__ as output_style_main +from colcon_core.output_style import add_output_style_arguments +from colcon_core.output_style import apply_output_style +from colcon_core.output_style import DEFAULT_OUTPUT_STYLE_ENVIRONMENT_VARIABLE +from colcon_core.output_style import OutputStyleExtensionPoint +from colcon_core.output_style import StyleCollection +from colcon_core.output_style import Stylizer +import pytest + +from .extension_point_context import ExtensionPointContext + + +MarkdownBold = Stylizer('**', '**') +MarkdownItalic = Stylizer('_', '_') + + +class LoudErrors(OutputStyleExtensionPoint): + + def apply_style(self, style): + style.Error = MarkdownBold + + +class SoftWarnings(OutputStyleExtensionPoint): + + PRIORITY = 90 + + def apply_style(self, style): + style.Warning = MarkdownItalic + + +def test_add_output_style_arguments(): + # No extensions available + parser = argparse.ArgumentParser() + with ExtensionPointContext(): + add_output_style_arguments(parser) + + # Secondary extensions are never the default + parser = argparse.ArgumentParser() + with ExtensionPointContext( + soft_warnings=SoftWarnings, + ): + add_output_style_arguments(parser) + + assert parser.get_default('output_style') is None + + # Default extension selection + parser = argparse.ArgumentParser() + with ExtensionPointContext( + loud_errors=LoudErrors, + soft_warnings=SoftWarnings, + ): + add_output_style_arguments(parser) + + assert parser.get_default('output_style') == 'loud_errors' + + # Environment variable selection + parser = argparse.ArgumentParser() + with ExtensionPointContext( + loud_errors=LoudErrors, + soft_warnings=SoftWarnings, + ): + with patch.dict( + 'colcon_core.output_style.os.environ', + {DEFAULT_OUTPUT_STYLE_ENVIRONMENT_VARIABLE.name: 'soft_warnings'}, + ): + add_output_style_arguments(parser) + + assert parser.get_default('output_style') == 'soft_warnings' + + +@patch('colcon_core.output_style.Style') +def test_apply_output_style(mock_class): + mock_class.Error = Stylizer.Default + with ExtensionPointContext( + loud_errors=LoudErrors, + soft_warnings=SoftWarnings, + ): + apply_output_style(SimpleNamespace(output_style=None)) + assert mock_class.Error('foo') == 'foo' + + apply_output_style(SimpleNamespace(output_style='loud_errors')) + assert mock_class.Error('foo') == '**foo**' + + apply_output_style(SimpleNamespace(output_style='soft_warnings')) + assert mock_class.Warning('foo') == '_foo_' + + with ExtensionPointContext( + unimplemented=OutputStyleExtensionPoint, + ): + with pytest.raises(NotImplementedError): + apply_output_style(SimpleNamespace(output_style='unimplemented')) + + +def test_combine_stylizers(): + bold_and_italic = MarkdownBold + MarkdownItalic + assert bold_and_italic('x') == '**_x_**' + + italic_and_bold = MarkdownItalic + MarkdownBold + assert italic_and_bold('X') == '_**X**_' + + with pytest.raises(TypeError): + MarkdownBold + 'x' + + +def test_style_default(): + style = StyleCollection() + assert style.DoesNotExist == Stylizer.Default + + +@patch('sys.stdout', new_callable=StringIO) +def test_style_dump(mock_stdout): + with ExtensionPointContext( + loud_errors=LoudErrors, + soft_warnings=SoftWarnings, + ): + runpy.run_path(output_style_main.__file__) + + stdout = mock_stdout.getvalue() + assert 'loud_errors' in stdout + assert 'soft_warnings' in stdout diff --git a/test/test_package_discovery.py b/test/test_package_discovery.py index 1533ac251..986fa2279 100644 --- a/test/test_package_discovery.py +++ b/test/test_package_discovery.py @@ -110,6 +110,7 @@ def test_discover_packages(): return_value={PackageDescriptor('/extension1/pkg1')}) extensions['extension2'].discover = Mock( return_value={PackageDescriptor('/extension2/pkg1')}) + extensions['extension2'].has_parameters = Mock(return_value=None) descs = discover_packages(None, None, discovery_extensions=extensions) assert len(descs) == 2 @@ -126,11 +127,13 @@ def test_discover_packages(): PackageDescriptor('/extension3/pkg2')}) descs = discover_packages(None, None, discovery_extensions=extensions) - assert len(descs) == 2 + assert len(descs) == 3 expected_path = '/extension3/pkg1'.replace('/', os.sep) assert expected_path in (str(d.path) for d in descs) expected_path = '/extension3/pkg2'.replace('/', os.sep) assert expected_path in (str(d.path) for d in descs) + expected_path = '/extension2/pkg1'.replace('/', os.sep) + assert expected_path in (str(d.path) for d in descs) def test_expand_dir_wildcards(): diff --git a/test/test_package_identification_python.py b/test/test_package_identification_python.py index 6390e24f6..600893ea6 100644 --- a/test/test_package_identification_python.py +++ b/test/test_package_identification_python.py @@ -74,7 +74,6 @@ def test_identify(): 'install_requires =\n' ' runA > 1.2.3\n' ' runB\n' - 'tests_require = test == 2.0.0\n' 'zip_safe = false\n' '[options.extras_require]\n' 'test = test2 == 3.0.0\n' @@ -93,7 +92,7 @@ def test_identify(): assert desc.dependencies['run'] == {'runA', 'runB'} dep = next(x for x in desc.dependencies['run'] if x == 'runA') assert dep.metadata['version_gt'] == '1.2.3' - assert desc.dependencies['test'] == {'test', 'test2', 'test3', 'test4'} + assert desc.dependencies['test'] == {'test2', 'test3', 'test4'} assert callable(desc.metadata['get_python_setup_options']) options = desc.metadata['get_python_setup_options'](None) diff --git a/test/test_pyproject_spec.py b/test/test_pyproject_spec.py new file mode 100644 index 000000000..072eb0a9d --- /dev/null +++ b/test/test_pyproject_spec.py @@ -0,0 +1,53 @@ +# Copyright 2024 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from colcon_core.package_descriptor import PackageDescriptor +from colcon_core.python_project.spec import load_and_cache_spec + + +def test_pyproject_missing(tmp_path): + desc = PackageDescriptor(tmp_path) + + spec = load_and_cache_spec(desc) + assert spec.get('build-system') == { + 'build-backend': 'setuptools.build_meta:__legacy__', + 'requires': ['setuptools >= 40.8.0', 'wheel'], + } + + +def test_pyproject_empty(tmp_path): + desc = PackageDescriptor(tmp_path) + + (tmp_path / 'pyproject.toml').write_text('') + + spec = load_and_cache_spec(desc) + assert spec.get('build-system') == { + 'build-backend': 'setuptools.build_meta:__legacy__', + 'requires': ['setuptools >= 40.8.0', 'wheel'], + } + + +def test_specified(tmp_path): + desc = PackageDescriptor(tmp_path) + + (tmp_path / 'pyproject.toml').write_text('\n'.join(( + '[build-system]', + 'build-backend = "my_build_backend.meta"', + 'requires = ["my-build-backend"]', + ))) + + spec = load_and_cache_spec(desc) + assert spec.get('build-system') == { + 'build-backend': 'my_build_backend.meta', + 'requires': ['my-build-backend'], + } + + # truncate the pyproject.toml and call again + # this verifies that the spec is cached + (tmp_path / 'pyproject.toml').write_text('') + + spec = load_and_cache_spec(desc) + assert spec.get('build-system') == { + 'build-backend': 'my_build_backend.meta', + 'requires': ['my-build-backend'], + } diff --git a/test/test_shell.py b/test/test_shell.py index 0c0378fb1..dc0db7e5b 100644 --- a/test/test_shell.py +++ b/test/test_shell.py @@ -19,6 +19,7 @@ from colcon_core.shell import get_command_environment from colcon_core.shell import get_environment_variables from colcon_core.shell import get_find_installed_packages_extensions +from colcon_core.shell import get_null_separated_environment_variables from colcon_core.shell import get_shell_extensions from colcon_core.shell import ShellExtensionPoint from colcon_core.shell.installed_packages import IsolatedInstalledPackageFinder @@ -156,6 +157,42 @@ async def check_output(cmd, **kwargs): assert 'DECODE_ERROR=' in warn.call_args[0][0] +def test_get_null_separated_environment_variables(): + cmd = [ + sys.executable, '-c', + r'import sys; sys.stdout.buffer.write(b"FOO\0NAME=value\nSOMETHING' + r'\0NAME2=value with spaces\0NAME3=NAME4\nNAME5=NAME6")'] + + coroutine = get_null_separated_environment_variables(cmd, shell=False) + env = run_until_complete(coroutine) + + assert len(env.keys()) == 3 + assert 'NAME' in env.keys() + assert env['NAME'] == 'value\nSOMETHING' + assert 'NAME2' in env.keys() + assert env['NAME2'] == 'value with spaces' + assert 'NAME3' in env.keys() + assert env['NAME3'] == 'NAME4\nNAME5=NAME6' + + # test with environment strings which isn't decodable + async def check_output(cmd, **kwargs): + return b'DECODE_ERROR=\x81\nNAME=value' + with patch('colcon_core.shell.check_output', side_effect=check_output): + with patch('colcon_core.shell.logger.warning') as warn: + coroutine = get_environment_variables(['not-used'], shell=False) + env = run_until_complete(coroutine) + + assert len(env.keys()) == 1 + assert 'NAME' in env.keys() + assert env['NAME'] == 'value' + # the raised decode error is catched and results in a warning message + assert warn.call_count == 1 + assert len(warn.call_args[0]) == 1 + assert warn.call_args[0][0].startswith( + "Failed to decode line from the environment using the encoding '") + assert 'DECODE_ERROR=' in warn.call_args[0][0] + + class Extension3(ShellExtensionPoint): PRIORITY = 105 diff --git a/test/test_shell_template.py b/test/test_shell_template.py index 2c6e04cf1..fc9604a90 100644 --- a/test/test_shell_template.py +++ b/test/test_shell_template.py @@ -42,6 +42,17 @@ def test_expand_template(): f" processing template '{template_path}'") assert not destination_path.exists() + # proper use + expand_template(template_path, destination_path, {'var': 'value1'}) + assert destination_path.exists() + assert destination_path.read_text() == 'value1' + # overwrite with a different value + expand_template(template_path, destination_path, {'var': 'value2'}) + assert destination_path.exists() + assert destination_path.read_text() == 'value2' + + destination_path.unlink() + # skip all symlink tests on Windows for now if sys.platform == 'win32': # pragma: no cover return diff --git a/test/test_spell_check.py b/test/test_spell_check.py index 10e5725d2..8ab0de481 100644 --- a/test/test_spell_check.py +++ b/test/test_spell_check.py @@ -2,6 +2,7 @@ # Licensed under the Apache License, Version 2.0 from pathlib import Path +import sys import pytest @@ -16,9 +17,14 @@ def known_words(): @pytest.mark.linter def test_spell_check(known_words): - from scspell import Report - from scspell import SCSPELL_BUILTIN_DICT - from scspell import spell_check + try: + from scspell import Report + from scspell import SCSPELL_BUILTIN_DICT + from scspell import spell_check + except ModuleNotFoundError: + if sys.version_info >= (3, 13): + pytest.skip('scspell3k is not available for Python 3.13+') + raise source_filenames = [ Path(__file__).parents[1] / 'bin' / 'colcon',