From 66c2d8de8963fadeb6dbb33a3871334e10162430 Mon Sep 17 00:00:00 2001 From: "Frederik F. Van der Veken" Date: Thu, 2 Jul 2026 09:57:21 +0200 Subject: [PATCH 1/6] WIP --- xobjects/context.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/xobjects/context.py b/xobjects/context.py index effd95a..965feb1 100644 --- a/xobjects/context.py +++ b/xobjects/context.py @@ -374,6 +374,36 @@ def get_installed_c_source_paths(self) -> List[str]: sources.append(str(path)) return sources + def get_installed_c_source_and_library_paths(self) -> List[str]: + """Returns a list of library paths registered in dependent packages. + + In a package that depends on xobjects, you can register C library + paths. This path, relative to the module, will be added to the + library path when building kernels. For example, the following will + allow to use functions from the library ``xcoll/lib/shared_lib.so``: + + .. code-block:: toml + [project.entry-points.xobjects] + libs = "FlukaIO" + lib_paths = "xcoll/lib" + """ + libs = [] + lib_paths = [] + sources = [] + for ep in entry_points(group="xobjects", name="build_info"): + get_build_info = ep.load() + info = get_build_info() + + include_dirs = info.get("include_dirs", []) + libraries = info.get("libraries", []) + library_dirs = info.get("library_dirs", []) + for ep in entry_points(group="xobjects", name="lib_paths"): + module = ep.load() + module_path = Path(module.__file__).parent + .parents[1] + libs.append(str(path)) + return libs + @abstractmethod def nparray_to_context_array(self, arr, copy=False): """Obtain an array on the context, given a numpy array. From 4a64ba69a1ed85b98717a3ea9ac30c878c5e43ca Mon Sep 17 00:00:00 2001 From: "Frederik F. Van der Veken" Date: Tue, 21 Jul 2026 16:43:27 +0200 Subject: [PATCH 2/6] Extended get_installed_c_source_paths into get_installed_c_source_and_library_paths to allow to declare C libraries via entry points --- xobjects/context.py | 104 +++++++++++++++++++++++------------ xobjects/context_cpu.py | 16 +++++- xobjects/context_cupy.py | 7 ++- xobjects/context_pyopencl.py | 7 ++- 4 files changed, 93 insertions(+), 41 deletions(-) diff --git a/xobjects/context.py b/xobjects/context.py index 965feb1..405ce87 100644 --- a/xobjects/context.py +++ b/xobjects/context.py @@ -354,55 +354,89 @@ def build_kernels( ) -> Dict[Tuple[str, tuple], KernelType]: pass - def get_installed_c_source_paths(self) -> List[str]: - """Returns a list of include paths registered in dependent packages. + def get_installed_c_source_and_library_paths(self) -> List[str]: + """Returns a list of C paths registered in dependent packages. - In a package that depends on xobjects, you can register C source paths - using the entry point `xobjects.c_sources`. A path to the directory - containing the specified module will be added to the include path when - building kernels. For example, the following will allow to write - ``#include `` in kernel sources: + In a package that depends on xobjects, you can register C source and + library paths using the entry point `xobjects.build_info`. These paths + will be added to the C include path and the library path when building + kernels. For example, the following will allow to write + ``#include `` in kernel sources, and + allow to use functions from the library ``xcoll/lib/libFlukaIO.a``: .. code-block:: toml [project.entry-points.xobjects] - include = "xtrack" + build_info = "xcoll._xobjects:get_build_info" + + and in the file ``xcoll/_xobjects.py``: + + .. code-block:: python + from ..general import _pkg_root + def get_build_info(): + return { + "include_dirs": [_pkg_root.parent], + "libraries": ["FlukaIO"], + "library_dirs": [_pkg_root / "lib"], + } """ - sources = [] + sources = set() + libs = set() + lib_paths = set() + + # Old entry point for backward compatibility for ep in entry_points(group="xobjects", name="include"): module = ep.load() path = Path(module.__file__).parents[1] - sources.append(str(path)) - return sources + sources.add(path) - def get_installed_c_source_and_library_paths(self) -> List[str]: - """Returns a list of library paths registered in dependent packages. - - In a package that depends on xobjects, you can register C library - paths. This path, relative to the module, will be added to the - library path when building kernels. For example, the following will - allow to use functions from the library ``xcoll/lib/shared_lib.so``: - - .. code-block:: toml - [project.entry-points.xobjects] - libs = "FlukaIO" - lib_paths = "xcoll/lib" - """ - libs = [] - lib_paths = [] - sources = [] + # New entry point for ep in entry_points(group="xobjects", name="build_info"): get_build_info = ep.load() info = get_build_info() - include_dirs = info.get("include_dirs", []) - libraries = info.get("libraries", []) + if not hasattr(include_dirs, "__iter__") \ + or isinstance(include_dirs, str): + include_dirs = [include_dirs] + include_dirs = [Path(dd).expanduser().resolve() + for dd in include_dirs] + sources.update(include_dirs) + library_dirs = info.get("library_dirs", []) - for ep in entry_points(group="xobjects", name="lib_paths"): - module = ep.load() - module_path = Path(module.__file__).parent - .parents[1] - libs.append(str(path)) - return libs + if not hasattr(library_dirs, "__iter__") \ + or isinstance(library_dirs, str): + library_dirs = [library_dirs] + library_dirs = [Path(dd).expanduser().resolve() + for dd in library_dirs] + lib_paths.update(library_dirs) + + libraries = info.get("libraries", []) + if not hasattr(libraries, "__iter__") \ + or isinstance(libraries, str): + libraries = [libraries] + libs.update(libraries) + + # Only add existing libraries + final_lib_paths = set() + for lib in list(libs): + found = False + for lib_path in lib_paths: + if (lib_path / f"lib{lib}.a").exists() \ + or (lib_path / f"lib{lib}.so").exists() \ + or (lib_path / f"lib{lib}.dylib").exists() \ + or (lib_path / f"{lib}.dll").exists() \ + or (lib_path / f"{lib}.dll.a").exists() \ + or (lib_path / f"{lib}.lib").exists(): + found = True + final_lib_paths.add(lib_path) + break + if not found: + log.warning( + f"Library {lib} not found in any of the library paths: " + f"{lib_paths}. It will be ignored." + ) + libs.remove(lib) + + return sources, libs, final_lib_paths @abstractmethod def nparray_to_context_array(self, arr, copy=False): diff --git a/xobjects/context_cpu.py b/xobjects/context_cpu.py index 4bf53a2..a9fac7d 100644 --- a/xobjects/context_cpu.py +++ b/xobjects/context_cpu.py @@ -442,10 +442,20 @@ def compile_kernel( xtr_compile_args.append("-DXO_CONTEXT_CPU_SERIAL") xtr_link_args.append("-DXO_CONTEXT_CPU_SERIAL") - extra_include_paths = self.get_installed_c_source_paths() - include_flags = [f"-I{path}" for path in extra_include_paths] + ( + extra_include_paths, + extra_library_paths, + extra_library_folders, + ) = self.get_installed_c_source_and_library_paths() + include_flags = [f"-I{path.as_posix()}" + for path in extra_include_paths] xtr_compile_args.extend(include_flags) xtr_link_args.extend(include_flags) + library_flags = [f"-L{path.as_posix()}" + for path in extra_library_paths] + xtr_link_args.extend(library_flags) + library_folders = [f"-l{lib}" for lib in extra_library_folders] + xtr_link_args.extend(library_folders) if os.name == "nt": # windows # TODO: to be handled properly @@ -456,6 +466,8 @@ def compile_kernel( xtr_compile_args.append("-w") xtr_link_args.append("-w") + print(f"{xtr_compile_args=}") + print(f"{xtr_link_args=}") ffi_interface.set_source( module_name, specialized_source, diff --git a/xobjects/context_cupy.py b/xobjects/context_cupy.py index 37d3fdd..aa601ef 100644 --- a/xobjects/context_cupy.py +++ b/xobjects/context_cupy.py @@ -463,8 +463,11 @@ def build_kernels( with open(save_source_as, "w") as fid: fid.write(specialized_source) - extra_include_paths = self.get_installed_c_source_paths() - include_flags = [f"-I{path}" for path in extra_include_paths] + ( + # TODO: how to deal with CUDA libraries? + extra_include_paths, _, _, + ) = self.get_installed_c_source_and_library_paths() + include_flags = [f"-I{path.as_posix()}" for path in extra_include_paths] extra_compile_args = ( *extra_compile_args, *include_flags, diff --git a/xobjects/context_pyopencl.py b/xobjects/context_pyopencl.py index 9538a3d..b0b61c6 100644 --- a/xobjects/context_pyopencl.py +++ b/xobjects/context_pyopencl.py @@ -228,8 +228,11 @@ def build_kernels( with open(save_source_as, "w") as fid: fid.write(specialized_source) - extra_include_paths = self.get_installed_c_source_paths() - include_flags = [f"-I{path}" for path in extra_include_paths] + ( + # TODO: how to deal with OpenCL libraries? + extra_include_paths, _, _, + ) = self.get_installed_c_source_and_library_paths() + include_flags = [f"-I{path.as_posix()}" for path in extra_include_paths] extra_compile_args = ( *extra_compile_args, From 6af270b1c5a8af79d182ba443c483d036bc3773e Mon Sep 17 00:00:00 2001 From: "Frederik F. Van der Veken" Date: Tue, 21 Jul 2026 16:55:29 +0200 Subject: [PATCH 3/6] Quick fix in library definition from entry point --- xobjects/context_cpu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xobjects/context_cpu.py b/xobjects/context_cpu.py index 0eb509d..b5c5d86 100644 --- a/xobjects/context_cpu.py +++ b/xobjects/context_cpu.py @@ -505,10 +505,10 @@ def compile_kernel( for path in extra_include_paths] xtr_compile_args.extend(include_flags) xtr_link_args.extend(include_flags) - library_flags = [f"-L{path.as_posix()}" - for path in extra_library_paths] + library_folders = [f"-L{path.as_posix()}" + for path in extra_library_folders] + library_flags = [f"-l{lib}" for lib in extra_library_paths] xtr_link_args.extend(library_flags) - library_folders = [f"-l{lib}" for lib in extra_library_folders] xtr_link_args.extend(library_folders) if os.name == "nt": # windows From ba8430ec26b0a1c4c91b2972916b1bc2d2b542ac Mon Sep 17 00:00:00 2001 From: "Frederik F. Van der Veken" Date: Thu, 23 Jul 2026 03:26:13 +0200 Subject: [PATCH 4/6] Do not remove libraries that are not existing - this is the developer's responsibility --- xobjects/context.py | 52 +++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/xobjects/context.py b/xobjects/context.py index 405ce87..bff5d46 100644 --- a/xobjects/context.py +++ b/xobjects/context.py @@ -354,7 +354,9 @@ def build_kernels( ) -> Dict[Tuple[str, tuple], KernelType]: pass - def get_installed_c_source_and_library_paths(self) -> List[str]: + def get_installed_c_source_and_library_paths( + self, + ) -> tuple[set[Path], set[str], set[Path]]: """Returns a list of C paths registered in dependent packages. In a package that depends on xobjects, you can register C source and @@ -394,49 +396,33 @@ def get_build_info(): get_build_info = ep.load() info = get_build_info() include_dirs = info.get("include_dirs", []) - if not hasattr(include_dirs, "__iter__") \ - or isinstance(include_dirs, str): + if not hasattr(include_dirs, "__iter__") or isinstance( + include_dirs, str + ): include_dirs = [include_dirs] - include_dirs = [Path(dd).expanduser().resolve() - for dd in include_dirs] + include_dirs = [ + Path(dd).expanduser().resolve() for dd in include_dirs + ] sources.update(include_dirs) library_dirs = info.get("library_dirs", []) - if not hasattr(library_dirs, "__iter__") \ - or isinstance(library_dirs, str): + if not hasattr(library_dirs, "__iter__") or isinstance( + library_dirs, str + ): library_dirs = [library_dirs] - library_dirs = [Path(dd).expanduser().resolve() - for dd in library_dirs] + library_dirs = [ + Path(dd).expanduser().resolve() for dd in library_dirs + ] lib_paths.update(library_dirs) libraries = info.get("libraries", []) - if not hasattr(libraries, "__iter__") \ - or isinstance(libraries, str): + if not hasattr(libraries, "__iter__") or isinstance( + libraries, str + ): libraries = [libraries] libs.update(libraries) - # Only add existing libraries - final_lib_paths = set() - for lib in list(libs): - found = False - for lib_path in lib_paths: - if (lib_path / f"lib{lib}.a").exists() \ - or (lib_path / f"lib{lib}.so").exists() \ - or (lib_path / f"lib{lib}.dylib").exists() \ - or (lib_path / f"{lib}.dll").exists() \ - or (lib_path / f"{lib}.dll.a").exists() \ - or (lib_path / f"{lib}.lib").exists(): - found = True - final_lib_paths.add(lib_path) - break - if not found: - log.warning( - f"Library {lib} not found in any of the library paths: " - f"{lib_paths}. It will be ignored." - ) - libs.remove(lib) - - return sources, libs, final_lib_paths + return sources, libs, lib_paths @abstractmethod def nparray_to_context_array(self, arr, copy=False): From 9df90d2cf4ce69be6a69b942fb70e79808bdfcdd Mon Sep 17 00:00:00 2001 From: "Frederik F. Van der Veken" Date: Thu, 23 Jul 2026 03:48:33 +0200 Subject: [PATCH 5/6] Use CFFI built-in arguments intead of xtra_compile_flags --- xobjects/context_cpu.py | 61 +++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/xobjects/context_cpu.py b/xobjects/context_cpu.py index b5c5d86..e26383c 100644 --- a/xobjects/context_cpu.py +++ b/xobjects/context_cpu.py @@ -24,11 +24,15 @@ def _class_allows_no_prebuilt_kernel(cls): return ( - getattr(cls, 'allow_no_prebuilt_kernel', False) - or getattr(getattr(cls, '_DressingClass', None), - 'allow_no_prebuilt_kernel', False) - or getattr(getattr(cls, '_XoStruct', None), - 'allow_no_prebuilt_kernel', False) + getattr(cls, "allow_no_prebuilt_kernel", False) + or getattr( + getattr(cls, "_DressingClass", None), + "allow_no_prebuilt_kernel", + False, + ) + or getattr( + getattr(cls, "_XoStruct", None), "allow_no_prebuilt_kernel", False + ) ) @@ -38,39 +42,38 @@ def allow_no_prebuilt_kernel_enabled(context=None, classes=()): elif isinstance(classes, type): classes = (classes,) - if os.environ.get('XSUITE_ALLOW_NO_PREBUILT_KERNELS') is not None: + if os.environ.get("XSUITE_ALLOW_NO_PREBUILT_KERNELS") is not None: return True if allow_no_prebuilt_kernel: return True if any(_class_allows_no_prebuilt_kernel(cls) for cls in classes): return True - return getattr(context, 'allow_no_prebuilt_kernel', False) + return getattr(context, "allow_no_prebuilt_kernel", False) def _is_serial_cpu_context(context): - if context is None or not hasattr(context, 'openmp_enabled'): + if context is None or not hasattr(context, "openmp_enabled"): return False return context.openmp_enabled is False def require_prebuilt_kernel(context=None, classes=()): - return ( - not allow_no_prebuilt_kernel_enabled(context, classes=classes) - and _is_serial_cpu_context(context) - ) + return not allow_no_prebuilt_kernel_enabled( + context, classes=classes + ) and _is_serial_cpu_context(context) def no_prebuilt_kernel_jit_message(): return ( - 'To allow just-in-time compilation instead, as in older Xsuite ' - 'versions, set the environment variable ' - '`XSUITE_ALLOW_NO_PREBUILT_KERNELS`, set ' - '`xobjects.context_cpu.allow_no_prebuilt_kernel = True`, or set ' - '`context.allow_no_prebuilt_kernel = True`. Classes that require ' - 'just-in-time compilation can also define ' - '`allow_no_prebuilt_kernel = True` as a class attribute. Using ' - 'just-in-time compilation instead of prebuilt kernels may require ' - 'lengthy compilation whenever a different kernel is needed.' + "To allow just-in-time compilation instead, as in older Xsuite " + "versions, set the environment variable " + "`XSUITE_ALLOW_NO_PREBUILT_KERNELS`, set " + "`xobjects.context_cpu.allow_no_prebuilt_kernel = True`, or set " + "`context.allow_no_prebuilt_kernel = True`. Classes that require " + "just-in-time compilation can also define " + "`allow_no_prebuilt_kernel = True` as a class attribute. Using " + "just-in-time compilation instead of prebuilt kernels may require " + "lengthy compilation whenever a different kernel is needed." ) @@ -498,18 +501,9 @@ def compile_kernel( ( extra_include_paths, + extra_libraries, extra_library_paths, - extra_library_folders, ) = self.get_installed_c_source_and_library_paths() - include_flags = [f"-I{path.as_posix()}" - for path in extra_include_paths] - xtr_compile_args.extend(include_flags) - xtr_link_args.extend(include_flags) - library_folders = [f"-L{path.as_posix()}" - for path in extra_library_folders] - library_flags = [f"-l{lib}" for lib in extra_library_paths] - xtr_link_args.extend(library_flags) - xtr_link_args.extend(library_folders) if os.name == "nt": # windows # TODO: to be handled properly @@ -520,11 +514,12 @@ def compile_kernel( xtr_compile_args.append("-w") xtr_link_args.append("-w") - print(f"{xtr_compile_args=}") - print(f"{xtr_link_args=}") ffi_interface.set_source( module_name, specialized_source, + include_dirs=[path.as_posix() for path in extra_include_paths], + libraries=list(extra_libraries), + library_dirs=[path.as_posix() for path in extra_library_paths], extra_compile_args=xtr_compile_args, extra_link_args=xtr_link_args, ) From 5d53f63787df1a1ea05a3cbd9a5d0d4e7fc5de15 Mon Sep 17 00:00:00 2001 From: "Frederik F. Van der Veken" Date: Thu, 23 Jul 2026 17:14:20 +0200 Subject: [PATCH 6/6] Keep linter happy --- xobjects/context_cupy.py | 14 ++++++++++---- xobjects/context_pyopencl.py | 14 ++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/xobjects/context_cupy.py b/xobjects/context_cupy.py index c7e8351..e5b86b7 100644 --- a/xobjects/context_cupy.py +++ b/xobjects/context_cupy.py @@ -356,7 +356,8 @@ def __invert__(self): return cupy.ndarray.__invert__(self._as_cupy()) -cudaheader: List[SourceType] = ["""\ +cudaheader: List[SourceType] = [ + """\ typedef signed int int32_t; //only_for_context cuda typedef signed short int16_t; //only_for_context cuda typedef signed char int8_t; //only_for_context cuda @@ -373,7 +374,8 @@ def __invert__(self): #define NULL nullptr #endif -"""] +""" +] def nplike_to_cupy(arr): @@ -479,9 +481,13 @@ def build_kernels( ( # TODO: how to deal with CUDA libraries? - extra_include_paths, _, _, + extra_include_paths, + _, + _, ) = self.get_installed_c_source_and_library_paths() - include_flags = [f"-I{path.as_posix()}" for path in extra_include_paths] + include_flags = [ + f"-I{path.as_posix()}" for path in extra_include_paths + ] extra_compile_args = ( *extra_compile_args, *include_flags, diff --git a/xobjects/context_pyopencl.py b/xobjects/context_pyopencl.py index b0b61c6..cee7bc4 100644 --- a/xobjects/context_pyopencl.py +++ b/xobjects/context_pyopencl.py @@ -44,7 +44,8 @@ from ._patch_pyopencl_array import _patch_pyopencl_array -openclheader: List[SourceType] = ["""\ +openclheader: List[SourceType] = [ + """\ #ifndef XOBJ_STDINT typedef long int64_t; typedef int int32_t; @@ -58,7 +59,8 @@ #ifndef NULL #define NULL 0L #endif -"""] +""" +] if _enabled: # order of base classes matters as it defines which __setitem__ is used @@ -230,9 +232,13 @@ def build_kernels( ( # TODO: how to deal with OpenCL libraries? - extra_include_paths, _, _, + extra_include_paths, + _, + _, ) = self.get_installed_c_source_and_library_paths() - include_flags = [f"-I{path.as_posix()}" for path in extra_include_paths] + include_flags = [ + f"-I{path.as_posix()}" for path in extra_include_paths + ] extra_compile_args = ( *extra_compile_args,