diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c73498..7568ad3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,17 @@ env: CTEST_NO_TESTS_ACTION: error on: + workflow_dispatch: + inputs: + logLevel: + description: 'Log level' + required: true + default: 'warning' + type: choice + options: + - info + - warning + - debug push: paths: - "**/CMakeLists.txt" @@ -13,6 +24,7 @@ on: - "**.f90" - "**.py" - ".github/workflows/ci.yml" + - "pyproject.toml" jobs: @@ -21,7 +33,7 @@ jobs: strategy: matrix: os: [ubuntu-24.04] - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] include: - os: macos-latest python-version: "3.10" @@ -29,7 +41,7 @@ jobs: runs-on: ${{ matrix.os }} env: - FC: gfortran-14 + FC: gfortran steps: - uses: actions/checkout@v4 @@ -37,6 +49,12 @@ jobs: with: python-version: ${{ matrix.python-version }} + - uses: fortran-lang/setup-fortran@v1 + id: setup-fortran + with: + compiler: gcc + version: 14 + - run: pip install .[tests,lint] - run: flake8 @@ -47,11 +65,12 @@ jobs: - run: pytest + - run: pip install meson # required for cmake tests - run: cmake --workflow --preset default windows: runs-on: windows-latest - timeout-minutes: 10 + timeout-minutes: 60 steps: - uses: msys2/setup-msys2@v2 @@ -80,6 +99,7 @@ jobs: - run: pytest -v + - run: pip install meson # required for cmake tests - run: cmake -B build env: CMAKE_GENERATOR: "MinGW Makefiles" diff --git a/.gitignore b/.gitignore index a6dce58..87bb71e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,26 @@ dist/ +build/ +cmake-build-*/ +CMakeFiles/ +CMakeCache.txt + __pycache__/ *.py[cod] *.so *.dylib *.egg-info/ + +# Local virtual environments +.venv/ +venv/ + +# Ninja / CMake logs +rules.ninja +.ninja_log +.ninja_deps + +# OS specific files +.DS_Store +Thumbs.db \ No newline at end of file diff --git a/README.md b/README.md index 3e351e9..53a18e2 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ See below for how to make these examples. Lowtran requires a Fortran compiler and CMake. We use `f2py` (part of `numpy`) to seamlessly use Fortran libraries from Python by special compilation of the Fortran library with auto-generated shim code. -For Python 3.12, make sure to install setuptools using pip, i.e., `pip install setuptools` and make sure meson is available on the system. +Make sure `meson` and the python devel headers & libraries are available on your system. If a Fortran compiler is not already installed, install Gfortran: diff --git a/pyproject.toml b/pyproject.toml index d4ad752..6c9b25b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,10 @@ [build-system] -requires = ["setuptools>=61.0.0", "wheel", "numpy"] -build-backend = "setuptools.build_meta" +requires = [ + "scikit-build-core>=0.9.0", + "numpy", + "meson" +] +build-backend = "scikit_build_core.build" [project] name = "lowtran" @@ -16,7 +20,6 @@ classifiers = [ "Programming Language :: Fortran", "Topic :: Scientific/Engineering :: Atmospheric Science" ] -dynamic = ["readme"] requires-python = ">=3.9" dependencies = ["numpy", "xarray", "python-dateutil"] @@ -24,8 +27,9 @@ dependencies = ["numpy", "xarray", "python-dateutil"] tests = ["pytest"] lint = ["flake8", "flake8-bugbear", "flake8-builtins", "flake8-blind-except", "mypy", "types-python-dateutil"] -[tool.setuptools.dynamic] -readme = {file = ["README.md"], content-type = "text/markdown"} +[project.readme] +file = "README.md" +content-type = "text/markdown" [tool.black] line-length = 100 @@ -34,3 +38,11 @@ line-length = 100 files = ["src", "example"] allow_redefinition = true ignore_missing_imports = true + +# scikit-build-core configuration: out-of-source build and package discovery +[tool.scikit-build] +cmake.args = ["-GNinja"] +editable.mode = "redirect" +build-dir = "build/{wheel_tag}" +wheel.packages = ["src/lowtran"] +sdist.include = ["src/lowtran", "README.md", "LICENSE.txt", "MANIFEST.in"] diff --git a/src/lowtran/base.py b/src/lowtran/base.py index 08658e6..b5e53a6 100644 --- a/src/lowtran/base.py +++ b/src/lowtran/base.py @@ -3,52 +3,76 @@ import xarray import numpy as np from typing import Any -from pathlib import Path -import importlib.util -import distutils.sysconfig +import importlib +import sysconfig import os +import sys +from pathlib import Path from types import ModuleType -from .cmake import build - def check() -> ModuleType: - try: - lowtran7 = import_f2py_mod("lowtran7") - except ImportError: - src = Path(__file__).parent - build(source_dir=src, build_dir=src / "build") - lowtran7 = import_f2py_mod("lowtran7") + """Ensure the compiled lowtran7 extension is available. - return lowtran7 + With scikit-build-core, the extension is built at install time and shipped in the wheel. + If it's missing, prompt the user to install the package (e.g., `pip install .`). + """ + try: + return import_f2py_mod("lowtran7") + except ImportError as e: + raise ImportError( + "lowtran7 extension not found. Please install the package so the Fortran " + "extension is built (e.g., `pip install .` or `pip install lowtran`)." + ) from e -def import_f2py_mod(name: str) -> ModuleType: - if os.name == "nt": - # https://github.com/space-physics/lowtran/issues/19 - # code inspired by scipy._distributor_init.py for loading DLLs on Window - dll_path = (Path(__file__) / "../build/lowtran7/.libs").resolve() - if dll_path.is_dir(): - # add the folder for Python 3.8 and above - logging.info(f"Adding {dll_path} to DLL search path") - os.add_dll_directory(dll_path) # type: ignore - else: - logging.info(f"Could not find {dll_path} to add to DLL search path") - - mod_name = name + distutils.sysconfig.get_config_var("EXT_SUFFIX") # type: ignore - mod_file = Path(__file__).parent / mod_name - if not mod_file.is_file(): - raise ModuleNotFoundError(mod_file) - spec = importlib.util.spec_from_file_location(name, mod_file) - if spec is None: - raise ModuleNotFoundError(f"{name} not found in {mod_file}") - mod = importlib.util.module_from_spec(spec) - if mod is None: - raise ImportError(f"could not import {name} from {mod_file}") - spec.loader.exec_module(mod) # type: ignore - - return mod +def import_f2py_mod(name: str) -> ModuleType: + lib_name = name + sysconfig.get_config_var("EXT_SUFFIX") + lib_path = importlib.resources.files(__package__) / lib_name + + if not lib_path.is_file(): + # Assume editable install: navigate up to find the build directory + src_dir = Path(__file__).parent + project_root = src_dir.parent.parent # Assuming src/lowtran structure + + # scikit-build-core build directory pattern + build_base = project_root / "build" + + if build_base.exists(): + # Recursively search for the library file + matches = list(build_base.rglob(lib_name)) + if matches: + lib_path = matches[0] # Use the first match + + if not lib_path.is_file(): + raise ModuleNotFoundError(f"Module not found: {lib_path}") + + # On Windows, add DLL search directories to fix loading issues + dll_dirs: list[Any] = [] + if sys.platform == "win32" and hasattr(os, 'add_dll_directory'): + # Add common locations where your dependencies might be, i.e., system PATH and module dir + search_paths = os.environ.get('PATH', '').split(os.pathsep) + search_paths.append(os.fspath(lib_path.parent)) + + for path in search_paths: + try: + dll_dirs.append(os.add_dll_directory(path)) + except OSError: + pass + try: + # Load the module from file path + spec = importlib.util.spec_from_file_location(name, lib_path) + if spec is None or spec.loader is None: + raise ImportError(f"Cannot load module from {lib_path}") + + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + finally: + # Clean up DLL directories + for dll_dir in dll_dirs: + dll_dir.close() def nm2lt7(short_nm: float, long_nm: float, step_cminv: float = 20) -> tuple[float, float, float]: diff --git a/src/lowtran/cmake.py b/src/lowtran/cmake.py deleted file mode 100644 index fc63de7..0000000 --- a/src/lowtran/cmake.py +++ /dev/null @@ -1,32 +0,0 @@ -import subprocess -import shutil -from pathlib import Path -import os -import logging - -__all__ = ["build"] - - -def build(source_dir: Path, build_dir: Path) -> None: - """build with CMake""" - cmake = shutil.which("cmake") - if not cmake: - raise FileNotFoundError("CMake not found. Try:\n pip install cmake") - - gen = os.environ.get("CMAKE_GENERATOR", "") - if not gen or "Visual Studio" in gen: - if shutil.which("ninja") or shutil.which("samu") or shutil.which("ninja-build"): - gen = "Ninja" - elif os.name == "nt" and shutil.which("mingw32-make"): - gen = "MinGW Makefiles" - else: - gen = "Unix Makefiles" - - # %% Configure - cmd = [cmake, f"-B{build_dir}", f"-S{source_dir}", f"-G{gen}"] - logging.info(" ".join(cmd)) - subprocess.check_call(cmd) - # %% Build - cmd = [cmake, "--build", str(build_dir), "--parallel"] - logging.info(" ".join(cmd)) - subprocess.check_call(cmd) diff --git a/src/lowtran/cmake/f2py.cmake b/src/lowtran/cmake/f2py.cmake index 05c12ec..997898c 100644 --- a/src/lowtran/cmake/f2py.cmake +++ b/src/lowtran/cmake/f2py.cmake @@ -1,10 +1,13 @@ # f2py -find_package(Python COMPONENTS Interpreter NumPy REQUIRED) +set(Python3_FIND_VIRTUALENV FIRST) +find_package(Python3 COMPONENTS Interpreter NumPy REQUIRED) +message(STATUS "${PYTHON3_EXECUTABLE}") +message(STATUS "${Python3_NumPy_VERSION}") if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU" AND - CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10 AND - Python_NumPy_VERSION VERSION_LESS 1.19) + CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10 AND + Python3_NumPy_VERSION VERSION_LESS 1.19) message(FATAL_ERROR "Numpy >= 1.19 required for GCC >= 10") endif() @@ -15,26 +18,13 @@ if(f2py_suffix) endif() execute_process( -COMMAND ${Python_EXECUTABLE} -c "import sysconfig; x=sysconfig.get_config_var('EXT_SUFFIX'); assert x is not None; print(x)" +COMMAND ${Python3_EXECUTABLE} -c "import sysconfig; x=sysconfig.get_config_var('EXT_SUFFIX'); assert x is not None; print(x)" OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE ret OUTPUT_VARIABLE out ERROR_VARIABLE err ) -if(NOT ret EQUAL 0) - -message(VERBOSE "${ret}: ${out}: ${err}") - -execute_process( -COMMAND ${Python_EXECUTABLE} -c "import distutils.sysconfig; x=distutils.sysconfig.get_config_var('EXT_SUFFIX'); assert x is not None; print(x)" -OUTPUT_STRIP_TRAILING_WHITESPACE -RESULT_VARIABLE ret -OUTPUT_VARIABLE out -ERROR_VARIABLE err -) - -endif() if(NOT ret EQUAL 0) message(FATAL_ERROR "${ret}: ${out}: ${err}: could not determine f2py output file suffix") diff --git a/src/lowtran/cmake/f2pyTarget.cmake b/src/lowtran/cmake/f2pyTarget.cmake index f7bfad5..ae7d834 100644 --- a/src/lowtran/cmake/f2pyTarget.cmake +++ b/src/lowtran/cmake/f2pyTarget.cmake @@ -3,9 +3,27 @@ include(${CMAKE_CURRENT_LIST_DIR}/f2py.cmake) function(f2py_target module_name module_src out_dir) -set(f2py_bin ${CMAKE_CURRENT_BINARY_DIR}/${module_name}${f2py_suffix}) +set(f2py_bin "${CMAKE_CURRENT_BINARY_DIR}/${module_name}${f2py_suffix}") + +# Workaround for f2py/meson not handling spaces in paths correctly: +# Create a copy/symlink in build directory and use relative path when source path contains spaces +string(FIND "${module_src}" " " _has_space) +if(_has_space GREATER -1) + get_filename_component(_src_name "${module_src}" NAME) + set(_src_link "${CMAKE_CURRENT_BINARY_DIR}/${_src_name}") + # On Windows, symlinks require admin privileges, so use copy instead + # On Unix, symlink is preferred to avoid duplication + if(WIN32) + configure_file("${module_src}" "${_src_link}" COPYONLY) + else() + file(CREATE_LINK "${module_src}" "${_src_link}" COPY_ON_ERROR SYMBOLIC) + endif() + set(_f2py_src "${_src_name}") +else() + set(_f2py_src "${module_src}") +endif() -set(f2py_arg -m ${module_name} -c ${module_src}) +set(f2py_arg -m ${module_name} -c ${_f2py_src} --backend meson) if(CMAKE_Fortran_COMPILER_ID MATCHES "^Intel") if(WIN32) list(APPEND f2py_arg --fcompiler=intelvem) @@ -15,15 +33,26 @@ if(CMAKE_Fortran_COMPILER_ID MATCHES "^Intel") endif() add_custom_command( -OUTPUT ${f2py_bin} -COMMAND ${f2py} ${f2py_arg} +OUTPUT "${f2py_bin}" +COMMAND "${f2py}" ${f2py_arg} +WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" +VERBATIM +DEPENDS "${module_src}" ) -add_custom_target(${module_name} ALL DEPENDS ${f2py_bin}) +add_custom_target(${module_name} ALL DEPENDS "${f2py_bin}") + +# Install the built Python extension into the wheel/lib directory during +# `cmake --install`, letting the build backend collect it. This avoids copying +# artifacts into the source tree. +if(DEFINED SKBUILD_PLATLIB_DIR) + set(_dest "${SKBUILD_PLATLIB_DIR}/lowtran") +else() + # Fallback: install under Python site-packages/lib directory name + include(GNUInstallDirs) + set(_dest "${CMAKE_INSTALL_LIBDIR}/lowtran") +endif() -add_custom_command( -TARGET ${module_name} POST_BUILD -COMMAND ${CMAKE_COMMAND} -E copy ${f2py_bin} ${out_dir}/ -) +install(FILES "${f2py_bin}" DESTINATION "${_dest}") endfunction(f2py_target) diff --git a/src/lowtran/tests/test_scenarios.py b/src/lowtran/tests/test_scenarios.py index 657ffc5..a86a6bc 100644 --- a/src/lowtran/tests/test_scenarios.py +++ b/src/lowtran/tests/test_scenarios.py @@ -36,7 +36,7 @@ def test_irradiance(): TR = lowtran.irradiance(c1) assert [c1["wllong"], c1["wlshort"]] == approx(TR.wavelength_nm[[0, -1]].values) - assert TR["transmission"][0, [0, 100], 0].values == approx([1.675140e-04, 0.2456177], rel=1e-6) + assert TR["transmission"][0, [0, 100], 0].values == approx([1.675140e-04, 0.2456177], rel=1e-5) assert TR["irradiance"][0, [100, 1000], 0].values == approx([0.00019873, 0.14551014], rel=1e-5) @@ -55,7 +55,7 @@ def test_radiance(): TR = lowtran.radiance(c1) assert [c1["wllong"], c1["wlshort"]] == approx(TR.wavelength_nm[[0, -1]].values) - assert TR["transmission"][0, [0, 100], 0].values == approx([1.675140e-04, 0.2456177], rel=1e-6) + assert TR["transmission"][0, [0, 100], 0].values == approx([1.675140e-04, 0.2456177], rel=1e-5) assert TR["radiance"][0, [10, 200], 0].values == approx([3.110389e-04, 3.907411e-10], rel=0.01)