diff --git a/.gitignore b/.gitignore index ff1d418d0..5fc88ccd7 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,5 @@ version.py # local directory idl_routines/ + +/uv.lock diff --git a/conftest.py b/conftest.py index d29ed8113..bd9248579 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,8 @@ +import sys +from pathlib import Path + import matplotlib as mpl mpl.use("Agg") + +sys.path.insert(0, str(Path(__file__).resolve().parent)) diff --git a/docs/bibliography.bib b/docs/bibliography.bib index 9c26a4eb2..59a01cde0 100644 --- a/docs/bibliography.bib +++ b/docs/bibliography.bib @@ -171,3 +171,23 @@ @ARTICLE{velasquez:2024 doi = {10.21105/joss.06396}, url = {https://doi.org/10.21105/joss.06396} } + +@ARTICLE{weber:2004, + author = {{Weber}, M.~A. and {DeLuca}, E.~E. and {Golub}, L. and {Sette}, A.~L.}, + title = "{Temperature diagnostics with multichannel imaging telescopes}", + journal = "{IAU Symposium 223: Multi-Wavelength Investigations of Solar Activity}", + year = 2004, + pages = {321--328}, + publisher = {Cambridge University Press}, + doi = {10.1017/S1743921304006088} +} + +@ARTICLE{golub:2004, + author = {{Golub}, L. and {DeLuca}, E.~E. and {Sette}, A. and {Weber}, M.}, + title = "{DEM analysis with the X-Ray Telescope (XRT) for Solar-B}", + journal = "{ASP Conference Series, The Solar-B Mission and the Outlook for Space-Based Solar Physics}", + year = 2004, + volume = {325}, + pages = {217--222}, + publisher = {Astronomical Society of the Pacific} +} diff --git a/docs/changelog/0.6.0.rst b/docs/changelog/0.6.0.rst new file mode 100644 index 000000000..b97fc2b73 --- /dev/null +++ b/docs/changelog/0.6.0.rst @@ -0,0 +1,48 @@ +XRTpy v0.6.0 +========================= + +New Features +------------ +- Introduced :class:`~xrtpy.xrt_dem_iterative.XRTDEMIterative`, a Python + implementation of the IDL routine ``xrt_dem_iterative2.pro``. This solver + uses spline-parameterized DEM curves and iterative least-squares fitting to + infer the differential emission measure from Hinode/XRT multi-filter + observations. (`#PR `__) +- Added Monte Carlo uncertainty estimation to the DEM solver through the + ``monte_carlo_runs`` parameter. Perturbed intensity realizations are used + to estimate the spread in DEM(T). +- Added support for user-supplied intensity uncertainties through the + ``intensity_errors`` parameter. If not provided, the default uncertainty is + ``max(0.03 * intensity, 2 DN/s/pix)``, matching the IDL behavior. +- Added ``plot_dem()`` and ``plot_dem_mc()`` methods to + :class:`~xrtpy.xrt_dem_iterative.XRTDEMIterative` for visualizing the base + DEM solution and Monte Carlo ensemble. +- Added a ``summary()`` method to + :class:`~xrtpy.xrt_dem_iterative.XRTDEMIterative` that reports the solver + configuration, inputs, and results. + +Documentation Updates +--------------------- +- Added a new DEM overview page introducing DEM analysis with Hinode/XRT, + describing the mathematical background, and providing worked examples + using :class:`~xrtpy.xrt_dem_iterative.XRTDEMIterative`. +- Added an API reference page for ``xrtpy.xrt_dem_iterative``. +- Updated the general documentation, including the XRT overview, getting + started guide, glossary, and documentation index. (`#398 + `__) + +Testing and Infrastructure +-------------------------- +- Enabled parallel test execution and improved pytest output and test + distribution settings. (`#400 + `__) +- Added Python 3.14 to the supported Python versions and continuous + integration test matrix. (`#401 + `__) + +Compatibility +------------- +- Requires Python 3.11 or later. +- Added support for Python 3.14. (`#401 + `__) +- Added dependencies on ``lmfit`` and ``scipy``. diff --git a/docs/changelog/index.rst b/docs/changelog/index.rst index 52fcc8764..30db6526a 100644 --- a/docs/changelog/index.rst +++ b/docs/changelog/index.rst @@ -9,6 +9,7 @@ This document lists the changes made during each release of XRTpy, including bug .. toctree:: :maxdepth: 1 + 0.6.0 0.5.1 0.5.0 0.4.0 diff --git a/docs/conf.py b/docs/conf.py index 941c35134..3158e2627 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,12 +4,16 @@ # -- stdlib imports ------------------------------------------------------------ import os +import sys import warnings from datetime import datetime, timezone from pathlib import Path from packaging.version import Version +# Ensure Sphinx imports the local xrtpy checkout (repo root), not site-packages +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + # -- Read the Docs Specific Configuration -------------------------------------- # This needs to be done before xrtpy is imported on_rtd = os.environ.get("READTHEDOCS", None) == "True" @@ -230,6 +234,7 @@ "feedback_communication*": [], "contributing*": [], "code_of_conduct*": [], + "dem_overview*": [], } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/dem_overview.rst b/docs/dem_overview.rst new file mode 100644 index 000000000..25cc3b637 --- /dev/null +++ b/docs/dem_overview.rst @@ -0,0 +1,321 @@ +.. _xrtpy-dem-overview: + +============================== +Differential Emission Measures +============================== + +.. contents:: + :local: + :depth: 2 + +Introduction +------------ + +This page describes the XRTpy iterative differential emission measure (DEM) solver and the inputs and outputs needed to run it. + +The DEM describes how much plasma is present +in the solar corona as a function of temperature along a line of sight. DEMs are key diagnostics for +understanding coronal heating, solar flares, and the thermal structure of +active regions. + +Hinode/XRT is well suited for DEM analysis because it observes the corona +through multiple broadband filters, each sensitive to different temperature +ranges. By combining these channels, we can infer a DEM(T) that reproduces +the observed filter intensities when forward-modeled with the instrument +temperature response functions. + +The solver is iterative in that it repeatedly adjusts a parameterized +DEM to minimize the difference between observed and modeled intensities. + +DEM in XRTpy +------------ +XRTpy provides a Python implementation of the iterative spline fitting method +originally available in IDL as `xrt_dem_iterative2.pro `__. +The core solver is implemented in :class:`xrtpy.xrt_dem_iterative.XRTDEMIterative`. + +Conceptually, the solver: + 1. Builds a regular grid in :math:`\log_{10}(T)` between user-specified bounds. + 2. Interpolates the filter temperature responses onto that grid. + 3. Represents :math:`\log_{10}(DEM)` as a spline in :math:`\log_{10}(T)`. + 4. Uses least-squares fitting (via ``lmfit``) to adjust the spline values so that the modeled filter intensities best match the observed intensities. + 5. Optionally performs Monte Carlo runs by perturbing the observed intensities with their uncertainties and re-solving the DEM many times to estimate uncertainties. + +This approach mirrors the structure and behavior of the IDL routine while providing a modern, +fully open-source implementation in Python that integrates naturally with the scientific Python ecosystem. + +Required inputs +--------------- +The DEM workflow requires three main input pieces: + +1. Observed channels (filters) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* Type: ``list`` of ``str`` +* Description: Names of the filters used in the observation, for example ``"Al-mesh"`` or ``"Be-thin"``. + +2. Temperature response functions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The DEM class requires temperature response functions for each filter, which +describe the instrument sensitivity as a function of temperature. These +responses can be generated for a chosen observation date using utilities +provided in `xrtpy.response.tools`. + +* Units: DN s\ :sup:`-1` pix\ :sup:`-1` cm\ :sup:`5` +* Description: Instrument response as a function of temperature for each filter, matching the order of the filters. +* Can be generated using :func:`xrtpy.response.tools.generate_temperature_responses`. + +Example +^^^^^^^ + +.. code-block:: python + + from xrtpy.response.tools import generate_temperature_responses + + filters = ["Al-poly", "C-poly", "Ti-poly"] + responses = generate_temperature_responses( + filters, + "2012-07-10T12:03:20", + ) + + +3. Observed intensities +~~~~~~~~~~~~~~~~~~~~~~~ +* Type: array-like +* Units: DN/s/pix +* Description: Measured intensities in each filter channel. +* Length must match the number of filters. + + +Overview of the XRTDEMIterative API +----------------------------------- +The main entry point is :class:`xrtpy.xrt_dem_iterative.XRTDEMIterative`. + +LogT grid construction +~~~~~~~~~~~~~~~~~~~~~~ +The solver constructs a uniformly spaced grid in :math:`\log_{10}(T)` between +``minimum_bound_temperature`` and ``maximum_bound_temperature`` using +``logarithmic_temperature_step_size``. The linear temperature grid is then computed as +:math:`T = 10^{\log_{10}(T)}` (Kelvin). + +.. rubric:: Notes + +This mirrors the "regular logT grid" option used by the IDL routine :file:`xrt_dem_iterative2.pro`. + +.. rubric:: Attributes created + +logT : `~numpy.ndarray` + The regular :math:`\log_{10}(T)` grid (dimensionless). + +T : `~astropy.units.Quantity` + The linear temperature grid in kelvin. + +dlogT : float + Step size in :math:`\log_{10}(T)`. + +dlnT : float + Step size in :math:`\ln(T)`, computed as ``np.log(10) * dlogT``. +Solving a DEM +~~~~~~~~~~~~~ +.. code-block:: python + + from xrtpy.response.tools import generate_temperature_responses + from xrtpy.xrt_dem_iterative import XRTDEMIterative + + filters = ["Al-poly", "Ti-poly", "Be-thin", "C-poly"] + intensities = [520.0, 104.0, 901.0, 458.0] # DN/s/pix + observation_date = "2012-10-27T10:00:03" + + responses = generate_temperature_responses(filters, observation_date) + + dem_solver = XRTDEMIterative( + observed_channel=filters, + observed_intensities=intensities, + temperature_responses=responses, + ) + + # Solve for the DEM + dem_solver.solve() # returns the DEM array, also stored in dem_solver.dem + + # Plot the DEM + dem_solver.plot_dem() + + # Access DEM results + dem = dem_solver.dem + logT = dem_solver.logT + + # DEM solution + print(dem_solver.dem) + + # Temperature grid + print(dem_solver.logT) + + + +Enabling Monte Carlo uncertainties estimates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +To estimate uncertainties, you can enable Monte Carlo iterations. The solver +will perturb the observed intensities by their uncertainties and re-solve the DEM +for each realization. + +.. code-block:: python + + N_mc = 100 # number of Monte Carlo runs + + dem_solver = XRTDEMIterative( + observed_channel=filters, + observed_intensities=intensities, + temperature_responses=responses, + monte_carlo_runs=N_mc, + ) + + dem_solver.solve() + + # Monte Carlo DEM Plot + dem_solver.plot_dem_mc() # base DEM plus Monte Carlo curves + +The arrays ``dem_solver.mc_dem``, ``dem_solver.mc_chisq``, +``dem_solver.mc_base_obs``, and ``dem_solver.mc_mod_obs`` are then available +for custom analysis. + +Comparison with IDL +------------------- +The Python solver is designed to closely follow the logic of the +SolarSoft/IDL routine `xrt_dem_iterative2.pro `__: + +* Uses a regular :math:`\log_{10}(T)` grid. +* Represents :math:`\log_{10}(\mathrm{DEM})` at a set of spline knots. +* Uses a least-squares algorithm to minimize the chi-square statistic. +* Supports Monte Carlo noise realizations for uncertainty estimation. + +Small numerical differences between the Python and IDL implementations can arise due to: + +* Different interpolation choices (``scipy.interpolate.CubicSpline`` with natural boundary conditions in Python versus IDL's tension spline with :math:`\sigma = 1.0`). +* Response interpolation: IDL uses linear interpolation (``interpol``) to place the temperature responses onto the solver grid, while XRTpy uses ``scipy.interpolate.CubicSpline``. This can produce small but systematic differences in the response matrix, particularly where the response curve changes rapidly with temperature. +* Differences in optimization libraries (`lmfit `__ versus IDL `MPFIT `__). +* Floating-point rounding and platform-specific details. + +Within these limits, the Python implementation is intended to produce +results that are consistent with the IDL tool. + +Users should expect the following when comparing XRTpy and IDL results: + +* **General agreement:** For well-constrained temperature bins (where the observed filters have strong sensitivity), the two implementations typically agree within a few percent. +* **Low-emission temperature ranges:** At temperatures where little plasma is present - particularly at the cool and hot ends of the grid — relative differences tend to be larger. In these bins the DEM is poorly constrained by the data and the two optimizers may converge to different local minima. +* **Response interpolation:** IDL uses linear interpolation (``interpol``) to place the temperature responses onto the solver grid, while XRTpy uses ``scipy.interpolate.CubicSpline``. This can produce small but systematic differences in the response matrix, particularly where the response curve changes rapidly with temperature. +* **Optimizer sensitivity:** Because ``lmfit`` and IDL's ``MPFIT`` use different convergence criteria and step-size heuristics, solutions can diverge in cases where the chi-square surface has multiple shallow minima, such as observations with a complex multi-temperature structure. +* **Monte Carlo spread as a guide:** A practical way to assess whether a difference between IDL and XRTpy is significant is to compare it against the Monte Carlo uncertainty spread. If the IDL solution falls within the XRTpy MC envelope (or vice versa), the difference is within the natural uncertainty of the inversion. + +Mathematical background +----------------------- +This section provides a short description of the equations solved by the +XRTpy DEM solver. It is intended for orientation rather than as a full +mathematical derivation. + +The DEM inversion problem is underdetermined — there are more temperature +bins than observed channels — meaning that multiple thermal distributions +can reproduce the same set of observations. For each filter channel :math:`i`, the +observed intensity :math:`I_i` is related to the +DEM by + +.. math:: + + I_i = \int DEM(T)\, R_i(T)\, dT + +where :math:`R_i(T)` is the temperature response function of the filter and +:math:`\mathrm{DEM}(T)` describes the amount of emitting plasma as a function of +temperature. + +Because the number of temperature bins typically exceeds the number of observed +channels, the inversion does not have a unique solution. XRTpy therefore uses a +forward-fitting approach. The DEM is represented as a smooth function in +:math:`\log_{10}(T)` using a small number of spline knots. Model intensities are +computed on a discrete temperature grid as + +.. math:: + + I_i^{model} = \sum_j DEM(T_j)\, R_i(T_j)\, T_j\, \Delta(\ln T) + +The spline values are adjusted to minimize the chi-square statistic + +.. math:: + + \chi^2 = \sum_i \left[ + \frac{I_i^{model} - I_i^{obs}}{\sigma_i} + \right]^2 + +where :math:`\sigma_i` are the observational uncertainties. Smoothness of the +solution is enforced implicitly through the spline representation and the +limited number of knots. + +When Monte Carlo uncertainties estimation is enabled, the observed intensities are +perturbed according to their uncertainties, + +.. math:: + + I_i^{(k)} = I_i^{obs} + \mathcal{N}(0, \sigma_i) + +and the DEM is re-fit for each realization. The spread of the resulting DEM +curves provides an estimate of the uncertainty in :math:`DEM(T)`. + + +Extended example with options +----------------------------- +Below is an extended example showing additional constructor options. +The values shown match the current defaults and are written out for clarity. + +.. code-block:: python + + from xrtpy.response.tools import generate_temperature_responses + from xrtpy.xrt_dem_iterative import XRTDEMIterative + + filters = ["Al-poly", "Ti-poly", "Be-thin", "C-poly"] + # Example intensities + intensities = [520.0, 104.0, 901.0, 458.0] # DN/s/pix + observation_date = "2012-10-27T10:00:03" + + responses = generate_temperature_responses( + filters, + observation_date, + ) + + dem_solver = XRTDEMIterative( + observed_channel=filters, # Filter names + observed_intensities=intensities, # Observed intensity values + temperature_responses=responses, # Instrument responses + # Optional configuration: + intensity_uncertainties=None, # Observed uncertainties - default: auto-estimated (3%) + minimum_bound_temperature=5.5, # Minimum log T (default: 5.5) + maximum_bound_temperature=8.0, # Maximum log T (default: 8.0) + logarithmic_temperature_step_size=0.1, # Bin width in log T (default: 0.1) + monte_carlo_runs=100, # Number of Monte Carlo runs (default: 0 -(disabled)) + max_iterations=2000, # Solver max iterations (default: 2000) + normalization_factor=1e21, # Normalization scaling factor (default: 1e21) + ) + + dem_solver.solve() + dem_solver.plot_dem_mc() + +.. note:: + The values shown above correspond to the solver defaults and are written + out here to illustrate which parameters can be tuned. You can adjust these + to suit your specific analysis needs. This mirrors the flexibility of the + IDL routine :file:`xrt_dem_iterative2.pro`. + +References +---------- +The following references provide background on the Hinode/XRT instrument and +its use in coronal diagnostics: + +- Golub, L., et al. (2004), Solar Physics, 243, 63. :cite:p:`golub:2004` +- Weber, M. A., et al. (2004), Astrophysical Journal, 605, 528. :cite:p:`weber:2004` + +Notes and warnings +------------------ +The solver performs basic validation of user inputs and may emit warnings in +cases where observed intensities are non-physical (e.g., negative values) or +approach known instrument limits. These warnings do not stop execution but are +intended to help users assess the reliability of the results. + +When no intensity uncertainties are provided, the solver applies a default +model: :math:`\sigma_i = \max(0.03 \times I_i,\ 2\ \mathrm{DN/s/pix})`. +This mirrors the default behavior of the IDL routine. diff --git a/docs/feedback_communication.rst b/docs/feedback_communication.rst index d1a3021c4..c908c6651 100644 --- a/docs/feedback_communication.rst +++ b/docs/feedback_communication.rst @@ -1,4 +1,4 @@ -.. _xrytpy-feedback-communication: +.. _xrtpy-feedback-communication: ****************** Providing Feedback diff --git a/docs/gallery/data_processing/README.txt b/docs/gallery/data_processing/README.txt new file mode 100644 index 000000000..c433c4852 --- /dev/null +++ b/docs/gallery/data_processing/README.txt @@ -0,0 +1,11 @@ +Advanced Data Processing +========================= + +This section covers advanced image correction and diagnostic tools. + +Examples here show how to: +- Deconvolve images to correct for blurring +- Identify and remove light leaks +- Estimate temperature and emission measure using multi-filter observations + +These are core XRTpy functionalities for high-level solar data analysis. diff --git a/docs/gallery/image_filtering/README.txt b/docs/gallery/image_filtering/README.txt new file mode 100644 index 000000000..05a0732aa --- /dev/null +++ b/docs/gallery/image_filtering/README.txt @@ -0,0 +1,6 @@ +Image Filtering & Visualization +=============================== + +This section includes examples that demonstrate how to sort, filter, and visualize XRT image data. + +The tools shown here help users prepare XRT observations for further scientific analysis. diff --git a/docs/gallery/image_filtering/__init__.py b/docs/gallery/image_filtering/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docs/gallery/image_filtering/sorting_data.py b/docs/gallery/image_filtering/sorting_data.py new file mode 100644 index 000000000..ed4fe0b98 --- /dev/null +++ b/docs/gallery/image_filtering/sorting_data.py @@ -0,0 +1,92 @@ +""" +========================= +Filtering and Visualizing +========================= + +This example provides a simple overview of filtering and visualizing XRT data. +""" + +import astropy.units as u +import matplotlib.pyplot as plt +import sunpy.map +from astropy.visualization import ImageNormalize, SqrtStretch +from sunpy.net import Fido +from sunpy.net import attrs as a + +############################################################################## +# To start we will download a range of XRT data from the Virtual Solar Observatory (VSO). +# The goal is to acquire a large set of files we can sort through and visualize. + +query = Fido.search( + a.Time("2021-05-21 18:51:00", "2021-05-22 00:00:00"), a.Instrument("xrt") +) +print(query) + +############################################################################## +# This query will return a large number of files, this is due to the fact we do not +# specify any additional filters. We can filter the data by specifying additional +# attributes in the query. +# +# For wavelength, we use a range that focuses the data to return only Al-Poly filter images. +# This will cut the results down in half. + +query = Fido.search( + a.Time("2021-05-21 20:51:00", "2021-05-22 00:00:00"), + a.Instrument("xrt"), + a.Wavelength(4 * u.nm, 5 * u.nm), +) +print(query) + +############################################################################## +# Now we will download the data. +# As this is still over 60 files, this process can take some time. + +xrt_files = Fido.fetch(query) + +############################################################################## +# We can now load the data into a `~sunpy.map.MapSequence` and create a animation. + +xrt_seq = sunpy.map.Map(xrt_files, sequence=True) + +fig = plt.figure() +ax = fig.add_subplot(projection=xrt_seq.maps[0]) +ani = xrt_seq.plot( + axes=ax, norm=ImageNormalize(vmin=0, vmax=5e3, stretch=SqrtStretch()) +) + +############################################################################## +# You might notice that there is a jump in the sequence. +# The size of the data and the pointing changes. +# We can exclude these images by filtering the data further. + +xrt_seq_filtered_shape = sunpy.map.Map( + [m for m in xrt_seq if m.data.shape == (384, 384)], sequence=True +) + +fig = plt.figure() +ax = fig.add_subplot(projection=xrt_seq.maps[0]) +ani = xrt_seq_filtered_shape.plot( + axes=ax, norm=ImageNormalize(vmin=0, vmax=5e3, stretch=SqrtStretch()) +) + +############################################################################## +# In fact, `sunpy.map.Map` provides many attributes that can be used to filter the data. +# This provides a lot of flexibility in how you can filter the data for your science objective. +# +# For example, we can filter the data by the exposure time or the detector. + +xrt_seq_filtered_exp_time = sunpy.map.Map( + [m for m in xrt_seq_filtered_shape if m.exposure_time < 0.1 * u.s], sequence=True +) + +fig = plt.figure() +ax = fig.add_subplot(projection=xrt_seq.maps[0]) +ani = xrt_seq_filtered_exp_time.plot( + axes=ax, norm=ImageNormalize(vmin=0, vmax=5e3, stretch=SqrtStretch()) +) + +############################################################################## +# If you want to save this animation to a file, you can use the ``save`` method. +# For more information on how to use this method, `see the matplotlib documentation `__. + +plt.show() diff --git a/docs/gallery/instrument_response/README.txt b/docs/gallery/instrument_response/README.txt new file mode 100644 index 000000000..162f04983 --- /dev/null +++ b/docs/gallery/instrument_response/README.txt @@ -0,0 +1,10 @@ +Instrument Response & Calibration +================================= + +These examples illustrate how to evaluate and interpret the XRT instrument's characteristics. + +You will learn to work with: +- The instrument's effective area across different filters +- Temperature response functions for solar plasma + +This section is helpful for understanding how the instrument "sees" the Sun at different temperatures. diff --git a/docs/gallery/instrument_response/channels.py b/docs/gallery/instrument_response/channels.py new file mode 100644 index 000000000..485fbfaba --- /dev/null +++ b/docs/gallery/instrument_response/channels.py @@ -0,0 +1,103 @@ +""" +============================= +Exploring XRT's Configuration +============================= + +This example explores the X-Ray Telescope (XRT) instrument properties +using XRTpy's `xrtpy.response.Channel`. It provides convenient methods and attributes +to access and analyze various aspects of the XRT instrument configuration. +""" + +import matplotlib.pyplot as plt + +import xrtpy + +############################################################################## +# We begin by defining a filter channel by its common abbreviation. +# In this example we will be exploring the titanium-on-polyimide filter. +# For detailed information about various filter channels and their characteristics, you can refer to :ref:`xrtpy-about-xrt-filters`. +# +# To explore the properties and characteristics of a defined filter channel, we will create a +# `xrtpy.response.Channel`. By passing in the filter name as an input, we can work +# with the properties associated with the titanium-on-polyimide filter. + +channel = xrtpy.response.Channel("Ti-poly") + +############################################################################## +# Now that we have created our channel, we can delve into the XRT instrument and its properties. +# We will start by examining basic information about the XRT instrument. + +print("Selected filter:", channel.name) +print("\nObservatory:", channel.observatory) +print("Instrument:", channel.instrument) + +############################################################################## +# It is important to note that most instrument properties of XRT remain the same +# regardless of the specific filter being used. This means that many characteristics +# and specifications of the XRT instrument, such as its dimensions, +# field of view, and detector properties, are independent of the selected filter. +# +# We can explore various characteristics of the the Charge-Coupled-Device (CCD) +# camera camera, such as its quantum efficiency and pixel size to list a few. + +print(channel.ccd.ccd_name) +print("\nPixel size: ", channel.ccd.ccd_pixel_size) +print("Full well: ", channel.ccd.ccd_full_well) +print("Gain left: ", channel.ccd.ccd_gain_left) +print("Gain right: ", channel.ccd.ccd_gain_right) +print("eV pre electron: ", channel.ccd.ccd_energy_per_electron) + +############################################################################## +# We can explore the XRT entrance filter properties utilizing ``entrancefilter``. + +print(channel.entrancefilter.entrancefilter_name) +print("Material: ", channel.entrancefilter.entrancefilter_material) +print("Thickness: ", channel.entrancefilter.entrancefilter_thickness) +print("Density: ", channel.entrancefilter.entrancefilter_density) + +############################################################################## +# XRT data is recorded through nine X-ray filters, which are implemented using two filter wheels. +# +# By utilizing the ``channel.filter_#`` notation, where ``#`` represents filter wheel 1 or 2, +# we can explore detailed information about the selected XRT channel filter. +# +# It's worth noting that sometimes the other filter will yield the result "Open," as it's not use. +# For more comprehensive information about the XRT filters, you can refer to :ref:`xrtpy-about-xrt-filters`. + +print("Filter Wheel:", channel.filter_2.filter_name) +print("\nFilter material:", channel.filter_2.filter_material) +print("Thickness: ", channel.filter_2.filter_thickness) +print("Density: ", channel.filter_2.filter_density) + +############################################################################## +# We can explore geometry factors in the XRT using ``geometry``. + +print(channel.geometry.geometry_name) +print("\nFocal length:", channel.geometry.geometry_focal_len) +print("Aperture Area:", channel.geometry.geometry_aperture_area) + +############################################################################## +# The XRT is equipped with two mirrors and We can access the properties of these +# mirrors using the ``channel_mirror_#`` notation, where ``#`` represents the +# first or second mirror surface. + +print(channel.mirror_1.mirror_name) +print("Material: ", channel.mirror_1.mirror_material) +print("Density: ", channel.mirror_1.mirror_density) +print("Graze_angle: ", channel.mirror_1.mirror_graze_angle) + +############################################################################## +# Finally we can explore the XRT transmission properties + +plt.figure() + +plt.plot(channel.wavelength, channel.transmission, label=f"{channel.name}") +plt.title(f"{channel.name} filter") +plt.xlabel(r"$\lambda$ [Å]") +plt.ylabel(r"Transmittance") +# The full range goes up to 400 Å, but we will limit it to 80 Å for better visualization +plt.xlim(0, 80) +plt.grid(color="lightgrey") +plt.tight_layout() + +plt.show() diff --git a/docs/gallery/instrument_response/temperature_response.py b/docs/gallery/instrument_response/temperature_response.py new file mode 100644 index 000000000..031d5153c --- /dev/null +++ b/docs/gallery/instrument_response/temperature_response.py @@ -0,0 +1,88 @@ +""" +==================== +Temperature Response +==================== + +In this example, we will explore the temperature response of the filters on XRT. +The temperature response provides important information on how XRT responds to +the different temperatures of X-ray emissions. +""" + +import matplotlib.pyplot as plt +import numpy as np + +import xrtpy + +############################################################################## +# A filter channel is defined by its common abbreviation, which represents +# a specific type of filter used to modify the X-ray radiation observed. +# In this example, we will explore the carbon-on-polyimide filter (abbreviated as "C-poly"). + +xrt_filter = "C-poly" + +############################################################################## +# `~.TemperatureResponseFundamental` provides the functions and properties for +# calculating the temperature response. + +date_time = "2023-09-22T21:59:59" +tpf = xrtpy.response.TemperatureResponseFundamental( + xrt_filter, date_time, abundance_model="Photospheric" +) + +############################################################################## +# To calculate the temperature response,we can do the following: + +temperature_response = tpf.temperature_response() +print("Temperature Response:\n", temperature_response) + +############################################################################## +# We will now visualize the temperature response function using CHIANTI. +# These temperatures are of the plasma and are independent of the channel filter. +# +# We use the log of the these temperatures, to enhance the visibility of the +# variations at lower temperatures. + +chianti_temperature = np.log10(tpf.CHIANTI_temperature.to_value()) + +############################################################################## +# Differences overtime arise from an increase of the contamination layer on the +# CCD which blocks some of the X-rays thus reducing the effective area. +# For detailed information about the calculation of the XRT CCD contaminant layer thickness, +# you can refer to +# `Montana State University `__. +# +# Additional information is provided by +# `Narukage et. al. (2011) `__. + + +launch_datetime = "2006-09-22T23:59:59" + +launch_temperature_response = xrtpy.response.TemperatureResponseFundamental( + xrt_filter, launch_datetime, abundance_model="Photospheric" +).temperature_response() + +############################################################################## +# Now we can plot the temperature response versus the log of the CHIANTI temperature +# and compare the results for the launch date and the chosen date. + +plt.figure() + +plt.plot( + chianti_temperature, + np.log10(temperature_response.value), + label=f"{date_time}", +) +plt.plot( + chianti_temperature, + np.log10(launch_temperature_response.value), + label=f"{launch_datetime}", + color="red", +) + +plt.title("XRT Temperature Response") +plt.xlabel("Log(T) ($K$)") +plt.ylabel("$DN$ $cm^5$ $ s^-1$ $pix^-1$") +plt.legend() +plt.grid() + +plt.show() diff --git a/docs/glossary.rst b/docs/glossary.rst index 178a371cf..02a000629 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -13,11 +13,31 @@ Glossary Channel A specific configuration of the XRT instrument, defined by the combination of filters in both filter wheels. Each channel affects the telescope's sensitivity and spectral response. + Chi-Square (:math:`\chi^2`) + A goodness-of-fit statistic that measures how well the modeled filter intensities match the observed intensities. It is computed as: + + .. math:: + + \chi^2 = \sum_i \left( \frac{I_i^{\mathrm{model}} - I_i^{\mathrm{obs}}}{\sigma_i} \right)^2 + + where :math:`I_i^{\mathrm{model}}` is the intensity predicted by the DEM forward + model for filter :math:`i`, :math:`I_i^{\mathrm{obs}}` is the observed intensity, + and :math:`\sigma_i` is the intensity uncertainty. A lower :math:`\chi^2` indicates + a better fit. In :class:`~xrtpy.xrt_dem_iterative.XRTDEMIterative`, the + :math:`\chi^2` of the observations is also a useful proxy for how well the recovered + DEM matches the true DEM. + Deconvolution A numerical image processing technique used to correct for the blurring caused by the telescope's Point Spread Function (PSF), improving sharpness and visibility of fine structures. DEM - Differential Emission Measure (DEM) — a function that describes the distribution of plasma as a function of temperature along the line of sight. XRTpy will support DEM modeling in future versions. + Differential Emission Measure (DEM) — a function that describes how much plasma is present along the line of sight as a function of temperature. See :ref:`xrtpy-dem-overview` for a detailed overview of DEM theory, + usage, and the solver provided in XRTpy. + + DEM Inversion + The process of determining the temperature distribution of coronal plasma (the DEM) from a small number of filter intensities. Since more temperature bins are used than available filters, the problem is mathematically + underconstrained ("ill posed"), so regularization and smoothing are required to obtain a stable, physical solution. + DN Data Number (DN) — the digital value recorded by the CCD, representing the detected photon flux, usually in DN s\ :sup:`−1`\ . @@ -38,11 +58,45 @@ Glossary Contamination (related to the XRT) Refers to the gradual accumulation of material on the CCD and focal plane filters (FPFs), which reduces instrument throughput. This time-dependent degradation impacts effective area calculations and must be accounted for in data analysis. Refer to Section 2.5.3 *Contamination* in the `SolarSoft XRT Analysis Guide`_ for more information. + + Monte Carlo DEM + A method for estimating DEM uncertainty through repeated calculations using randomly perturbed input data. Each DEM solution is computed by + adding random noise, based on the intensity uncertainties, to the observed intensities and then repeating the DEM inversion. The spread + of the resulting solutions estimates the uncertainty in the DEM at each temperature. In :class:`~xrtpy.xrt_dem_iterative.XRTDEMIterative`, the number of + realizations is controlled by the ``monte_carlo_runs`` parameter. + PSF Point Spread Function — describes the response of the telescope to a point source of light. In XRTpy, it is used in deconvolution routines to sharpen images. + Response matrix + A two-dimensional array containing the temperature response of each XRT + filter, interpolated onto the solver's regular :math:`\log_{10}(T)` + temperature grid. This matrix connects the DEM to the modeled filter + intensities through the forward model: + + .. math:: + + I_i^{\mathrm{model}} + = \sum_j \mathrm{DEM}(T_j)\,R_i(T_j)\,T_j\,\Delta(\ln T) + + Solar Emission Spectra - Emission spectra produced by solar plasma across a range of temperatures, calculated using spectral models such as CHIANTI. These spectra are used in temperature response and filter ratio methods + Emission spectra produced by solar plasma across a range of temperatures, calculated using spectral models such as CHIANTI. These spectra are used in temperature response and filter ratio methods. + + Spline Knots + Selected temperature locations used to parameterize the DEM curve in + :class:`~xrtpy.xrt_dem_iterative.XRTDEMIterative`. The DEM is represented + by a cubic spline, with the DEM values at a small number of knot locations + in :math:`\log_{10}(T)` space optimized during fitting. The number of knots + is set to + + .. math:: + + \min\left(N_{\mathrm{filters}} - 1,\, 7\right), + + which limits the number of fitted spline parameters relative to the + available observational constraints and matches the behavior of the IDL + routine ``xrt_dem_iterative2.pro``. Temperature Response The expected response of the instrument to isothermal plasma as a function of temperature, given in units of DN cm\ :sup:`5` s\ :sup:`−1` pix\ :sup:`−1` for each filter channel. Calculated using CHIANTI atomic models and user-defined abundances. diff --git a/docs/index.rst b/docs/index.rst index b2812ead1..56fe4fe08 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,6 +14,15 @@ Welcome to the documentation for **XRTpy** (version |version|) — a Python_ pac XRTpy is designed for solar physicists, astronomers, students, and anyone curious about the Sun's dynamic outer atmosphere. Whether you're conducting research or just beginning to explore the world of X-ray solar imaging, XRTpy gives you the tools to study the hot plasma of the solar corona using real space-based observations. +.. note:: + + **New in v0.6.0:** XRTpy now includes + :class:`~xrtpy.xrt_dem_iterative.XRTDEMIterative`, a Python implementation + of the IDL routine ``xrt_dem_iterative2.pro`` for computing Differential + Emission Measures from Hinode/XRT observations. See :ref:`xrtpy-dem-overview` + and the :ref:`changelog ` for details. + + .. toctree:: :maxdepth: 1 :caption: Contents: @@ -21,6 +30,7 @@ Whether you're conducting research or just beginning to explore the world of X-r about_xrt install getting_started + dem_overview generated/gallery/index reference/index acknowledging_xrtpy @@ -30,3 +40,21 @@ Whether you're conducting research or just beginning to explore the world of X-r contributing code_of_conduct changelog/index + +Published Work +-------------- + +The following paper describes the XRTpy package and its initial release- v0.4.0: + +:cite:p:`velasquez:2024` + + +Get Involved +------------ + +XRTpy is an open-source project and welcomes contributions from the community. + +* **Source code:** `github.com/HinodeXRT/xrtpy `__ +* **Report a bug or request a feature:** `GitHub Issues `__ +* **Contribute:** See the :ref:`contributing guide ` +* **Questions or feedback:** See the :ref:`feedback page ` diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 8882898b3..3b1941e3d 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -5,8 +5,10 @@ API Reference ************* .. toctree:: - :maxdepth: 1 - :caption: Contents: - :glob: + :maxdepth: 2 - * + xrtpy + response + util + xrt_dem_iterative + image_correction diff --git a/docs/reference/response.rst b/docs/reference/response.rst index b171b1c13..206bf6dcb 100644 --- a/docs/reference/response.rst +++ b/docs/reference/response.rst @@ -4,3 +4,11 @@ .. automodapi:: xrtpy.response :include-all-objects: + +Tools +===== + +.. automodule:: xrtpy.response.tools + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/reference/xrt_dem_iterative.rst b/docs/reference/xrt_dem_iterative.rst new file mode 100644 index 000000000..a952f65b9 --- /dev/null +++ b/docs/reference/xrt_dem_iterative.rst @@ -0,0 +1,6 @@ +*********************** +xrtpy.xrt_dem_iterative +*********************** + +.. automodapi:: xrtpy.xrt_dem_iterative + :include-all-objects: diff --git a/examples/dem_iterative_example.py b/examples/dem_iterative_example.py new file mode 100644 index 000000000..10fe8ad59 --- /dev/null +++ b/examples/dem_iterative_example.py @@ -0,0 +1,79 @@ +""" +Iterative DEM with Hinode/XRT +============================= + +This example shows how to compute a differential emission measure (DEM) +from Hinode/XRT multi-filter observations using the XRTpy iterative solver. + +Inputs +------ +- Filter names +- Observation date (for temperature response computation) +- Observed intensities in DN/s/pix +- Intensity uncertainties in DN/s/pix (optional) +""" + +from xrtpy.response.tools import generate_temperature_responses +from xrtpy.xrt_dem_iterative import XRTDEMIterative + +# %% +# 1) Choose filters + observation date +filters = ["Al-mesh", "Al-poly", "Be-thin", "Be-thick", "Al-med", "Al-poly/Ti-poly"] +observation_date = "2021-07-20T16:04" + +# %% +# 2) Provide observed intensities (DN/s/pix) +# Replace these with values measured from your data (per filter). +intensities = [ + 362.382339, # Al-mesh + 181.407312, # Al-poly + 148.933070, # Be-thin + 0.002636, # Be-thick + 1.459830, # Al-med + 45.390125, # Al-poly/Ti-poly +] + + +# %% +# 3) Generate temperature response functions for the selected filters +responses = generate_temperature_responses(filters, observation_date) + +# %% +# 4) Run the DEM solver (base solution) +dem_solver = XRTDEMIterative( + observed_channel=filters, + observed_intensities=intensities, + temperature_responses=responses, + # You may optionally provide ``intensity_uncertainties`` in DN/s/pix. + # If not provided, a default model is used: max(0.03 * intensity, 2 DN/s/pix) + # intensity_uncertainties=[56.7, 3.8, 121.7, 7.6, 15.2, 0.2], # example values +) + +dem_solver.solve() + +# %% +# 5) Inspect results +dem_solver.summary() + +# Access results directly for your own analysis: +dem = dem_solver.dem # DEM(T) array [cm^-5 K^-1] +logT = dem_solver.logT # log10 temperature grid [K] + +# %% +# 6) Plot DEM +dem_solver.plot_dem() + +# %% +# 7) Monte Carlo uncertainty estimate (optional) +# Re-run the solver with Monte Carlo to estimate DEM uncertainties. +# Each run perturbs the observed intensities by their uncertainties +# and re-solves, producing an ensemble of DEM curves. +dem_solver_mc = XRTDEMIterative( + observed_channel=filters, + observed_intensities=intensities, + temperature_responses=responses, + monte_carlo_runs=100, +) + +dem_solver_mc.solve() +dem_solver_mc.plot_dem_mc() diff --git a/pyproject.toml b/pyproject.toml index 1d39c68fe..6b9901897 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,22 @@ package-data.xrtpy = [ "data/*" ] [tool.setuptools_scm] write_to = "xrtpy/version.py" +[tool.ruff] +target-version = "py311" +lint.ignore = [] +lint.per-file-ignores."xrtpy/xrt_dem_iterative/**/*.py" = [ + "B905", + "BLE001", + "C408", + "F841", + "FBT003", + "ICN001", + "PLC0415", + "PT006" +] +lint.per-file-ignores."xrtpy/xrt_dem_iterative/test/utils_sav_io.py" = [ "N815", "PERF203" ] +lint.per-file-ignores."xrtpy/xrt_dem_iterative/utils_sav_io.py" = [ "N815", "PERF203" ] + [tool.codespell] ignore-words-list = """ 4rd, diff --git a/xrtpy/xrt_dem_iterative/__init__.py b/xrtpy/xrt_dem_iterative/__init__.py new file mode 100644 index 000000000..9b6ba442c --- /dev/null +++ b/xrtpy/xrt_dem_iterative/__init__.py @@ -0,0 +1,9 @@ +""" +XRT DEM Iterative Solver +""" + +from .dem_solver import XRTDEMIterative + +__all__ = [ + "XRTDEMIterative", +] diff --git a/xrtpy/xrt_dem_iterative/dem_plotting.py b/xrtpy/xrt_dem_iterative/dem_plotting.py new file mode 100644 index 000000000..aa2b519fe --- /dev/null +++ b/xrtpy/xrt_dem_iterative/dem_plotting.py @@ -0,0 +1,224 @@ +__all__ = [ + "plot_dem", + "plot_dem_mc", +] + +import matplotlib.pyplot as plt +import numpy as np + + +def _build_filter_str(filters, max_filters=6): + """ + Return a filter subtitle string, truncating gracefully if there are many filters. + """ + if not filters: + return None + if len(filters) <= max_filters: + return ", ".join(filters) + shown = ", ".join(filters[:max_filters]) + return f"{shown}, ... (+{len(filters) - max_filters} more)" + + +def _uncertainty_note(solver): + """ + Return a small footnote string if the default uncertainty model is active, + or None if the user supplied their own uncertainties. + """ + if getattr(solver, "_intensity_uncertainties", None) is not None: + return None + rel = getattr(solver, "relative_uncertainty", 0.03) + floor = getattr(solver, "min_observational_uncertainty", None) + floor_val = floor.value if floor is not None else 2.0 + return ( + f"Uncertainties: default model — " + f"max({int(rel * 100)}% \u00d7 I, {floor_val:.1f} DN/s)" + ) + + +def plot_dem(solver): + """ + Plot the base DEM solution in log10 space (no Monte Carlo needed). + + Parameters + ---------- + solver : XRTDEMIterative + Fully initialized DEM solver. + """ + if not hasattr(solver, "dem"): + solver.solve() + + logT = solver.logT + dem = np.asarray(solver.dem, dtype=float) + + dem_safe = np.clip(dem, 1e-40, None) + log_dem = np.log10(dem_safe) + + # Title + filter_str = _build_filter_str(getattr(solver, "filter_names", None)) + if filter_str: + title = f"Hinode/XRT \u2014 Base DEM Solution\nFilters: {filter_str}" + else: + title = "Hinode/XRT \u2014 Base DEM Solution" + + fig, ax = plt.subplots(figsize=(8, 5)) + + ax.step( + logT, + log_dem, + where="mid", + color="black", + linewidth=2.0, + label="Base DEM", + ) + + ax.set_xlabel(r"$\log_{10}\,T$ [K]") + ax.set_ylabel(r"$\log_{10}$ DEM [cm$^{-5}$ K$^{-1}$]") + ax.set_title(title) + ax.grid(visible=True, alpha=0.3) + ax.set_xlim(logT.min(), logT.max()) + + # y-limits based on finite values + finite = np.isfinite(log_dem) + if np.any(finite): + y = log_dem[finite] + pad = 0.25 * (y.max() - y.min() + 1e-6) + ax.set_ylim(y.min() - pad, y.max() + pad) + + ax.legend() + + # Uncertainty footnote - only shown when default model is active + note = _uncertainty_note(solver) + if note: + fig.text( + 0.5, + 0.01, + note, + ha="center", + va="bottom", + fontsize=8, + color="gray", + style="italic", + ) + plt.tight_layout(rect=[0, 0.05, 1, 1]) + else: + plt.tight_layout() + + plt.show() + + +def plot_dem_mc( + solver, + mc_color="black", + base_color="#1E90FF", + alpha_mc=0.18, + lw_mc=1.0, + lw_base=2.0, + figsize=(9, 6), +): + """ + Plot DEM with Monte Carlo ensemble (if present). + + - Base DEM: thick Dodger Blue step curve + - MC realizations: thin transparent black step curves + - Falls back gracefully to base-only if no MC results are present + """ + if not hasattr(solver, "dem"): + solver.solve() + + logT = solver.logT + base_dem = np.asarray(solver.dem, dtype=float) + base_safe_dem = np.clip(base_dem, 1e-100, None) + log_base_dem = np.log10(base_safe_dem) + + has_mc = hasattr(solver, "mc_dem") and solver.mc_dem is not None + if has_mc: + mc_dem = np.asarray(solver.mc_dem, dtype=float) # shape (N+1, n_T) + N = max(0, mc_dem.shape[0] - 1) + else: + mc_dem = None + N = 0 + + # Title - filter string truncated to avoid overflow + filter_str = _build_filter_str(getattr(solver, "filter_names", None)) + if filter_str: + title = f"Hinode/XRT \u2014 DEM with Monte Carlo\nFilters: {filter_str}" + else: + title = "Hinode/XRT \u2014 DEM with Monte Carlo" + + if N > 0: + title += f" ({N} MC realizations)" + + fig, ax = plt.subplots(figsize=figsize) + + # Plot MC curves first (so base DEM renders on top) + mc_handle = None + if has_mc and N > 0: + for i in range(1, N + 1): + dem_i = np.clip(mc_dem[i, :], 1e-100, None) + (line,) = ax.step( + logT, + np.log10(dem_i), + where="mid", + color=mc_color, + alpha=alpha_mc, + linewidth=lw_mc, + ) + if i == 1: + mc_handle = line # keep only one handle for the legend + + # Base DEM on top + (base_handle,) = ax.step( + logT, + log_base_dem, + where="mid", + color=base_color, + linewidth=lw_base, + label="Base DEM", + ) + + # Legend + if has_mc and N > 0 and mc_handle is not None: + mc_handle.set_alpha(1.0) # full opacity for the legend swatch + ax.legend( + handles=[base_handle, mc_handle], + labels=["Base DEM", f"MC realizations (N={N})"], + ) + else: + ax.legend(handles=[base_handle], labels=["Base DEM"]) + + ax.set_title(title) + ax.set_xlabel(r"$\log_{10}\,T$ [K]") + ax.set_ylabel(r"$\log_{10}$ DEM [cm$^{-5}$ K$^{-1}$]") + ax.grid(visible=True, alpha=0.3) + ax.set_xlim(logT.min(), logT.max()) + + # y-limits - log_base_dem is already in log space + logs = [log_base_dem] + if has_mc and N > 0: + logs.append(np.log10(np.clip(mc_dem[1:, :], 1e-100, None)).ravel()) + logs_all = np.concatenate(logs) + + finite = np.isfinite(logs_all) + if np.any(finite): + y = logs_all[finite] + pad = 0.25 * (y.max() - y.min() + 1e-6) + ax.set_ylim(y.min() - pad, y.max() + pad) + + # Uncertainty footnote - only shown when default model is active + note = _uncertainty_note(solver) + if note: + fig.text( + 0.5, + 0.01, + note, + ha="center", + va="bottom", + fontsize=8, + color="gray", + style="italic", + ) + plt.tight_layout(rect=[0, 0.05, 1, 1]) + else: + plt.tight_layout() + + plt.show() diff --git a/xrtpy/xrt_dem_iterative/dem_solver.py b/xrtpy/xrt_dem_iterative/dem_solver.py new file mode 100644 index 000000000..463cffd99 --- /dev/null +++ b/xrtpy/xrt_dem_iterative/dem_solver.py @@ -0,0 +1,1214 @@ +__all__ = [ + "XRTDEMIterative", +] + +import warnings + +import astropy.units as u +import numpy as np +from lmfit import Parameters, minimize +from scipy.interpolate import CubicSpline, interp1d +from xrtpy.util.filters import validate_and_format_filters +from xrtpy.xrt_dem_iterative import dem_plotting + + +class XRTDEMIterative: + """ + Differential Emission Measure (DEM) solver for Hinode/XRT observations. + + This class implements a Python version of the IDL routine + ``xrt_dem_iterative2.pro``, using spline-parameterized DEM curves and + iterative least-squares fitting. It supports Monte Carlo uncertainty analysis + and closely mirrors the logic of the original IDL algorithm. + + Parameters + ---------- + observed_channel : str or list of str, required + Names of the filters used in the observation (for example, + "Al-mesh", "Be-thin"). Must correspond one-to-one with the + temperature_responses argument. + observed_intensities : array-like, required + Observed intensities for each filter channel. Units are DN/s/pix. + temperature_responses : list, required + List of TemperatureResponseFundamental objects matching the filters. + Units are DN s^-1 pix^-1 cm^5. These can be created using + xrtpy.response.tools.generate_temperature_responses(). + intensity_uncertainties : array-like, optional + Uncertainties in the observed intensities. If None, a default model + is used: max(0.03 * intensity, 2 DN/s/pix). + minimum_bound_temperature : float, optional + Minimum value of the log10(T) grid. Default is 5.5. + maximum_bound_temperature : float, optional + Maximum value of the log10(T) grid. Default is 8.0. + logarithmic_temperature_step_size : float, optional + Step size for the log10(T) grid. Default is 0.1. + monte_carlo_runs : int, optional + Number of Monte Carlo repetitions to perform. Default is 0 (disabled). + max_iterations : int, optional + Maximum number of function evaluations for lmfit. Default is 2000. + normalization_factor : float, optional + Internal scaling factor used during optimization. Default is 1e21. + + Notes + ----- + - All lists (observed_channel, observed_intensities, temperature_responses) must be the same length. + - The log10(T) range must lie inside the native temperature grid provided by all filter responses. + - If intensity_uncertainties is not provided, a default model is used to estimate uncertainties. + """ + + # NOTE: If any default parameter values are changed below, update the + # corresponding example in docs/dem_overview.rst (Extended example section) + # to keep the documentation consistent. + + def __init__( + self, + observed_channel, + observed_intensities, + temperature_responses, + intensity_uncertainties=None, + minimum_bound_temperature=5.5, + maximum_bound_temperature=8.0, + logarithmic_temperature_step_size=0.1, + monte_carlo_runs=0, + max_iterations=2000, + normalization_factor=1e21, + ): + """ + Notes + ----- + - All input lists (``observed_channel``, ``observed_intensities``, and ``temperature_responses``) + must be the same length. Each entry should correspond to one filter. + + - The temperature grid range (``minimum_bound_temperature``, ``maximum_bound_temperature``) must lie entirely within the + response temperature ranges for all filters provided. + + - If ``intensity_uncertainties`` is not provided, a model-based uncertainty estimate is used: + max(0.03 * observed_intensity, 2 DN/s/pix), as in the IDL original. + + - Default XRT filter names include: + Al-mesh, Al-poly, C-poly, Ti-poly, Be-thin, Be-med, Al-med, Al-thick, Be-thick, + Al-poly/Al-mesh, Al-poly/Ti-poly, Al-poly/Al-thick, Al-poly/Be-thick. + """ + + # Validate and store filter names + if observed_channel is None or ( + hasattr(observed_channel, "__len__") and len(observed_channel) == 0 + ): + raise ValueError("`observed_channel` is required and cannot be empty.") + self.observed_channel = validate_and_format_filters(observed_channel) + + # Store intensity and uncertainty arrays + if observed_intensities is None or len(observed_intensities) == 0: + raise ValueError("`observed_intensities` is required and cannot be empty.") + self._observed_intensities = np.asarray(observed_intensities, dtype=float) + + if not np.all(np.isfinite(self._observed_intensities)): + raise ValueError("`observed_intensities` must be finite numbers.") + + # uncertainties + if intensity_uncertainties is not None: + self._intensity_uncertainties = np.asarray( + intensity_uncertainties, dtype=float + ) + else: + self._intensity_uncertainties = None + + # Store temperature grid parameters + self._logarithmic_temperature_step_size = float( + logarithmic_temperature_step_size + ) + self._minimum_bound_temperature = float(minimum_bound_temperature) + self._maximum_bound_temperature = float(maximum_bound_temperature) + if not (self._minimum_bound_temperature < self._maximum_bound_temperature): + raise ValueError( + "minimum_bound_temperature must be < maximum_bound_temperature." + ) + + # Check logarithmic_temperature_step_size is positive + if self._logarithmic_temperature_step_size <= 0: + raise ValueError( + "logarithmic_temperature_step_size must be a positive scalar." + ) + + n_pts = ( + int( + np.floor( + (self._maximum_bound_temperature - self._minimum_bound_temperature) + / logarithmic_temperature_step_size + + 1e-9 + ) + ) + + 1 + ) + if n_pts < 4: + raise ValueError("Temperature grid must have at least 4 points.") + + # Validate Monte Carlo setting + if isinstance(monte_carlo_runs, bool): + raise TypeError( + "monte_carlo_runs must be a non-negative whole number, not a boolean." + ) + elif ( + isinstance(monte_carlo_runs, int | np.integer) + or isinstance(monte_carlo_runs, float) + and monte_carlo_runs.is_integer() + ): + self._monte_carlo_runs = int(monte_carlo_runs) + else: + raise ValueError( + "monte_carlo_runs must be a non-negative whole number (e.g., 0, 1, 100). " + "Decimal values are not allowed." + ) + + if self._monte_carlo_runs < 0: + raise ValueError("monte_carlo_runs must be ≥ 0.") + + # Validate max_iterations + if not isinstance(max_iterations, int | np.integer) or max_iterations <= 0: + raise ValueError("max_iterations must be a positive integer.") + + self._max_iterations = int(max_iterations) + + # Store temperature response objects + self.responses = temperature_responses + + if temperature_responses is None or len(temperature_responses) == 0: + raise ValueError("`temperature_responses` is required and cannot be empty.") + + # Validate that the temperature grid falls within the responses + for r in self.responses: + logT_grid = np.log10(r.temperature.to_value(u.K)) + if not ( + self._minimum_bound_temperature >= logT_grid.min() + and self._maximum_bound_temperature <= logT_grid.max() + ): + raise ValueError( + f"The specified temperature range [{minimum_bound_temperature}, {maximum_bound_temperature}] is outside the bounds of one or more filter response grids.\n" + "Please ensure the temperature range fits within all responses.\n" + "Hint: Default response range is logT = 5.5 to 8.0. You can view each response's logT range via: [r.temperature for r in responses]" + ) + + # Check consistency between inputs + if not ( + len(self._observed_intensities) + == len(self.responses) + == len(self.observed_channel) + ): + raise ValueError( + f"\nLength mismatch in inputs:\n" + f" Observed intensities: {len(self._observed_intensities)}\n" + f" Responses: {len(self.responses)}\n" + f" Filter channels: {len(self.observed_channel)}\n" + ) + + try: + value = float(normalization_factor) + except (TypeError, ValueError) as err: + raise ValueError(f"Invalid normalization_factor: {err}") from err + + if value <= 0: + raise ValueError("normalization_factor must be a positive number.") + + self._normalization_factor = value + + self._using_estimated_uncertainty = ( + False # track whether default uncertainty model has been used + ) + + def validate_inputs(self) -> None: + """ + Validate user-provided inputs. Raises ValueError on any issue. + + Intentionally separate from __init__ so tests and users can construct + the object first, then explicitly validate (matches test expectations). + """ + # 1) observed_channel non-empty + if self.observed_channel is None or len(self.observed_channel) == 0: + raise ValueError("`observed_channel` is required and cannot be empty.") + + # 2) intensities: length & finite + if self._observed_intensities is None or len(self._observed_intensities) == 0: + raise ValueError("`observed_intensities` is required and cannot be empty.") + if not np.all(np.isfinite(self._observed_intensities)): + raise ValueError("`observed_intensities` must be finite numbers.") + + # 3) responses present + if self.responses is None or len(self.responses) == 0: + raise ValueError("`temperature_responses` is required and cannot be empty.") + + # 4) lengths consistent between filters/intensities/responses + if not ( + len(self._observed_intensities) + == len(self.responses) + == len(self.observed_channel) + ): + raise ValueError( + "Length mismatch: intensities, responses, and observed_channel must match." + ) + + # 5) temperature grid sanity + if not (self._minimum_bound_temperature < self._maximum_bound_temperature): + raise ValueError( + "minimum_bound_temperature must be < maximum_bound_temperature." + ) + if self._logarithmic_temperature_step_size <= 0: + raise ValueError( + "logarithmic_temperature_step_size must be a positive scalar." + ) + n_pts = ( + int( + np.floor( + (self._maximum_bound_temperature - self._minimum_bound_temperature) + / self._logarithmic_temperature_step_size + + 1e-9 + ) + ) + + 1 + ) + if n_pts < 4: + raise ValueError("Temperature grid must have at least 4 points.") + + # 6) grid range inside every response + for r in self.responses: + # logT_grid = np.log10(r.temperature.value) + logT_grid = np.log10(r.temperature.to_value(u.K)) + if not ( + self._minimum_bound_temperature >= logT_grid.min() + and self._maximum_bound_temperature <= logT_grid.max() + ): + raise ValueError( + f"The specified temperature range [{self._minimum_bound_temperature}, {self._maximum_bound_temperature}] " + "is outside the bounds of one or more filter response grids." + ) + + # 7) intensity_uncertainties length & finiteness (only if provided) + if self._intensity_uncertainties is not None: + if self._intensity_uncertainties.shape != self._observed_intensities.shape: + raise ValueError( + "Length of intensity_uncertainties must match observed_intensities." + ) + if not np.all(np.isfinite(self._intensity_uncertainties)) or np.any( + self._intensity_uncertainties < 0 + ): + raise ValueError("`intensity_uncertainties` must be finite and >= 0.") + + if np.all(self._observed_intensities == 0): + warnings.warn( + "\n\n All observed intensities are zero. DEM solution will yield zero. " + "Object created, but solving will return DEM=0. \n\n", + stacklevel=2, + ) + + if np.any(self._observed_intensities < 0): + bad = self._observed_intensities < 0 + bad_channels = [ + ch for ch, m in zip(self.observed_channel, bad, strict=False) if m + ] + bad_values = self._observed_intensities[bad] + + warnings.warn( + ( + "\n\nOne or more observed intensities are negative.\n" + "XRT intensity values are expected to be non-negative. " + "The solver will continue.\n\n" + f"Affected filters: {bad_channels}\n" + f"Affected intensities (DN): {bad_values}\n" + ), + category=UserWarning, + stacklevel=2, + ) + + # warn if any value is >= 2500 + if np.any(self._observed_intensities >= 2500): + bad = self._observed_intensities >= 2500 + bad_channels = [ + ch for ch, m in zip(self.observed_channel, bad, strict=False) if m + ] + bad_values = self._observed_intensities[bad] + + warnings.warn( + ( + "\n\nOne or more observed intensities are >= 2500 DN.\n" + "Note: XRT intensities can be provided as raw counts (DN) or as " + "exposure-normalized rates (DN/s). If your values are in DN/s, this " + "threshold may not apply and you can disregard this warning.\n\n" + "If your values are raw counts (DN): the Hinode/XRT CCD output linearity " + "is specified as deviation less than 1% up to 3000 DN (12-bit ADU). " + "Values near or above this limit may be non-linear or saturated and " + "could affect DEM results.\n" + 'See XRT Blue Book, Table 2.4-3 and Section 2.4.5 "Charge Spreading":\n' + "https://xrt.cfa.harvard.edu/resources/documents/XRT_BlueBook/SolarB_bluebook.XRT.pdf\n\n" + f"Affected filters: {bad_channels}\n" + f"Affected intensities (DN): {bad_values}\n" + ), + category=UserWarning, + stacklevel=2, + ) + + return None + + def __repr__(self): + return ( + f"" + ) + + @property + def observed_intensities( + self, + ) -> u.Quantity: + """ + Observed intensities with physical units. + + Returns + ------- + `~astropy.units.Quantity` + Intensities in DN/s/pix for each filter channel. + Where "pix" means a one-arcsecond, full -resolution XRT pixel. + """ + return self._observed_intensities * (u.DN / u.s) + + @property + def filter_names(self): + """ + List of filter names from the temperature responses. + """ + return [r.filter_name for r in self.responses] + + @property + def response_temperatures(self): + """ + List of temperature grids (K) for each filter response. + """ + return [r.temperature for r in self.responses] + + @property + def response_values(self): + """ + List of response values (DN cm^5 / pix / s) for each filter. + """ + return [r.response for r in self.responses] + + @property + def minimum_bound_temperature(self): + """ + Lower bound of log10 temperature grid. + """ + return self._minimum_bound_temperature + + @property + def maximum_bound_temperature(self): + """ + Upper bound of log10 temperature grid. + """ + return self._maximum_bound_temperature + + @property + def logarithmic_temperature_step_size(self): + """ + Bin width of log10 temperature grid. + """ + return self._logarithmic_temperature_step_size + + @property + def min_observational_uncertainty(self): + """ + Default - Minimum absolute observational intensity uncertainty applied to DN/s/pix when intensity uncertainty is not provided. + """ + return 2 * (u.DN / u.s) + + @property + def relative_uncertainty(self): + """ + Relative uncertainty used to scale intensity if an uncertainty is not provided. + Default is 0.03 (3%). + """ + return 0.03 + + @property + def intensity_uncertainties(self) -> u.Quantity: + """ + Return the intensity uncertainty values. + + If the user supplied intensity_uncertainties, those values are returned. + Otherwise a default model is used: + + sigma = max(0.03 * intensity, 2 DN/s/pix) + + This behavior mirrors the default uncertainty logic of the IDL routine + xrt_dem_iterative2.pro. + + `~astropy.units.Quantity` + Intensity uncertainties in DN/s for each filter. + + For details, see: + https://hesperia.gsfc.nasa.gov/ssw/hinode/xrt/idl/util/xrt_dem_iterative2.pro + """ + if self._intensity_uncertainties is not None: + return self._intensity_uncertainties * (u.DN / u.s) + + if not self._using_estimated_uncertainty: + warnings.warn( + ( + "\n\n No intensity_uncertainties provided. Using default model: " + "max(relative-uncertainty * observed_intensity, min_observational_uncertainty)\n" + f"=> relative_uncertainty = {self.relative_uncertainty}, " + f"min_observational_uncertainty = {self.min_observational_uncertainty.value} DN/s\n" + "See: https://hesperia.gsfc.nasa.gov/ssw/hinode/xrt/idl/util/xrt_dem_iterative2.pro\n\n" + ), + category=UserWarning, + stacklevel=2, # 1 + ) + + # JoyRemove - April 24,2026 + # if not self._using_estimated_uncertainty: + # print( + # "\n" + # + "=" * 72 + # + "\nWARNING: No intensity uncertainties were provided.\n" + # "Using the default uncertainty model:\n" + # f" relative_uncertainty = {self.relative_uncertainty}\n" + # f" min_observational_uncertainty = " + # f"{self.min_observational_uncertainty.value} DN/s\n" + "=" * 72 + "\n" + # ) + + self._using_estimated_uncertainty = True + + # Fixed in units + estimated = np.maximum( + (self.relative_uncertainty * self._observed_intensities) * (u.DN / u.s), + self.min_observational_uncertainty, + ) + return estimated + + @property + def monte_carlo_runs(self) -> int: + """ + Return + ------ + int + Number of Monte Carlo runs to perform (0 = disabled). + """ + return self._monte_carlo_runs + + @property + def normalization_factor(self): + """ + Scaling factor used during DEM optimization to stabilize the spline fit. + Corresponds to ``normalization_factor`` in IDL (default 1e21). + """ + return self._normalization_factor + + @property + def max_iterations(self): + """ + Maximum number of iterations allowed in the least-squares DEM solver + (e.g., when using `lmfit.minimize `__). Default is 2000. + """ + return self._max_iterations + + def create_logT_grid(self): + """ + Construct the regular log10 temperature grid used for DEM calculations. + + This builds a uniformly spaced grid in :math:`\\log_{10}(T)` between + ``minimum_bound_temperature`` and ``maximum_bound_temperature``, using + ``logarithmic_temperature_step_size``. The linear temperature grid is then + computed as :math:`T = 10^{\\log_{10}(T)}` in Kelvin. + + Notes + ----- + This mirrors the "regular logT grid" used by the IDL routine + ``xrt_dem_iterative2.pro``. + + Attributes created + ------------------ + logT : `~numpy.ndarray` + The regular :math:`\\log_{10}(T)` grid (dimensionless). + T : `~astropy.units.Quantity` + The linear temperature grid in Kelvin. + dlogT : float + Step size in :math:`\\log_{10}(T)`. + dlnT : float + Step size in :math:`\\ln(T)`, computed as ``np.log(10) * dlogT``. + """ + # number of bins including endpoints - if default values are used - end value is 26 + self.n_bins = ( + int( + round( + (self._maximum_bound_temperature - self._minimum_bound_temperature) + / self._logarithmic_temperature_step_size + ) + ) + + 1 + ) + + # This matches the IDL temperature grid exactly. self.logT & self.T. + # inclusive logT grid (IDL-style regular grid) + # Units = 'log K. Runs from minimum_bound_temperature to Max_T with bin-width = DT + # SELFNOTEJOY- Do we need to add units - current holds no units- it's working correctly - Should this on the Test as well?- I don't think it- it's noted in IDL but used with units + + # np.linspace over np.arange - simple reproduces that reliably:endpoint included, exact number of bins, and no accumulating floating-point drift - Best match to IDL + self.logT = np.linspace( + self._minimum_bound_temperature, + self._maximum_bound_temperature, + self.n_bins, + ) + + # linear temperature grid in Kelvin + self.T = (10.0**self.logT) * u.K + + self.dlogT = float( + self._logarithmic_temperature_step_size + ) # scalar spacing (dimensionless and natural-log equivalent) + + self.dlnT = ( + np.log(10.0) * self.dlogT + ) # for IDL-style integral DEM(T) * R(T) * T dlnT - IDL "regular logT grid" + + def _interpolate_responses_to_grid(self): + """ + Interpolate all filter responses onto the solver's regular log10(T) grid. + + This constructs the response matrix used in the DEM forward model. + Each filter's native temperature response (R(T)) is interpolated to the + grid defined by self.logT. Extrapolated values outside the native + response range are set to zero. + + Equivalent to the ``Res_Mat`` construction in the IDL routine xrt_dem_iterative2.pro. + + Notes + ----- + - Response units (from XRTpy) are DN s^-1 pix^-1 cm^5. + - Output matrix has shape (n_filters, n_temperatures). + - Rows correspond to filters; columns correspond to temperature bins. + + IDL method of Interpolate emissivity. + Interpolate all filter responses onto the common logT grid and build + the response matrix. Equivalent to constructing ``Res_Mat`` in IDL's ``xrt_dem_iterative2.pro``. + + Attributes Created + ------------------ + interpolated_responses : list of ndarray + _response_matrix : ndarray + Stacked filter responses on the uniform logT grid. + """ + if not hasattr(self, "logT"): + raise AttributeError( + "Temperature grid missing. Call create_logT_grid() first." + ) + + rows = [] + for T_orig, R_orig in zip( + self.response_temperatures, self.response_values, strict=False + ): # Make sure that R_orig.value is indeed in DN/s/pix per cm^5 + logT_orig = np.log10(T_orig.to_value(u.K)) + + response_vals = R_orig.value + + interp_func = interp1d( + logT_orig, + response_vals, + kind="linear", + bounds_error=False, + fill_value=0.0, + assume_sorted=True, + ) + rows.append(interp_func(self.logT)) + + self.interpolated_responses = rows + #self._response_matrix = np.vstack(rows).astype(float) #JOY-UPDATED-JULY16 + self._response_matrix = np.maximum(np.vstack(rows).astype(float), 0.0) + + # Store the physical unit for clarity + self._response_unit = (u.DN / u.s / u.pix) * u.cm**5 + + # Quick sanity check + if self._response_matrix.shape != (len(self.responses), self.logT.size): + raise RuntimeError("Interpolated response matrix has unexpected shape.") + + @property + def response_matrix(self): + """ + Response matrix (n_filters x n_temperatures) after interpolation. + + Units: DN s^-1 pix^-1 cm⁵ per emission measure. + + Equivalent to ``Res_Mat`` in IDL's ``xrt_dem_iterative2.pro``. + + Raises + ------ + AttributeError + If ``_interpolate_responses_to_grid()`` has not been called yet. + """ + if not hasattr(self, "_response_matrix"): + raise AttributeError( + "Response matrix not available. Call _interpolate_responses_to_grid() first." + ) + return self._response_matrix + + # DEM INITIALIZATION AND SOLVER METHODS + + def _estimate_initial_dem(self, cutoff: float = 1.0 / np.e) -> np.ndarray: + """ + Compute an initial DEM estimate closely following the structure of the IDL routine xrt_dem_iter_estim. + + The IDL code performs a rough DEM estimate by evaluating intensities + relative to response peaks, but xrt_dem_iterative2 ultimately replaces + that estimate with a flat log10(DEM) curve before calling the solver. + + This method repeats the peak-finding logic for diagnostic purposes, but + the final DEM passed into the solver is always: + + log10(DEM(T)) = 1.0 for all temperature bins + + which corresponds to DEM(T) = 1 in arbitrary units. This reproduces the + IDL initial condition exactly. + + Parameters + ---------- + cutoff : float, optional + Fraction of the peak response used to define the usable window + around a channel's emissivity peak. Default is 1/e (approximately + 0.3679). + + Returns + ------- + ndarray + Initial log10(DEM) estimate on self.logT. This is always a flat + array of ones (IDL-equivalent behavior). + """ + if not hasattr(self, "logT"): + raise AttributeError( + "Temperature grid missing. Call create_logT_grid() first." + ) + if not hasattr(self, "_response_matrix"): + raise AttributeError( + "Response matrix missing. Call _interpolate_responses_to_grid() first." + ) + + # Optional: store the peak-based estimates for diagnostics only. + # These are NOT used to set the initial DEM (IDL overwrites them + # with a flat DEM before handing off to the solver). + t_peaks = [] + log_dem_estimates = [] + + # Loop over each filter/channel with non-zero intensity + for T_orig, R_orig, I_obs in zip( + self.response_temperatures, + self.response_values, + self._observed_intensities, + strict=False, + ): + logT_orig = np.log10(T_orig.to_value(u.K)) + # Make sure the response is in DN s^-1 pix^-1 per EM (cm^-5) + R_vals = R_orig.to_value((u.DN / u.s / u.pix) * u.cm**5) + + if I_obs <= 0 or np.all(R_vals <= 0): + continue # skip unusable channel + + # 1. Peak location (logT) + max_idx = np.argmax(R_vals) + peak_val = R_vals[max_idx] + t_peak_raw = logT_orig[max_idx] + + # Round to nearest grid step in logT, similar to round_off(..., 0.1) + step = self._logarithmic_temperature_step_size + t_peak = np.round(t_peak_raw / step) * step + + # 2. Good window (where R > cutoff * peak) + good = np.where(R_vals > peak_val * cutoff)[0] + if good.size < 1: + continue + + # 3. Compute denominator integral: sum(T * R * dlnT) + T_good = 10.0 ** logT_orig[good] # [K] + R_good = R_vals[good] + # Native spacing in log10(T) + if logT_orig.size > 1: + dlogT_native = np.mean(np.diff(logT_orig)) + else: + # Degenerate case; fall back to solver grid spacing + dlogT_native = step + dlnT_native = np.log(10.0) * dlogT_native + denom = np.sum(T_good * R_good * dlnT_native) + + if denom <= 0: + continue + + # 4. DEM estimate at the peak (for diagnostics only) + dem_peak = I_obs / denom # [cm^-5 K^-1] + if dem_peak <= 0 or not np.isfinite(dem_peak): + continue + + log_dem_est = np.log10(dem_peak) + t_peaks.append(t_peak) + log_dem_estimates.append(log_dem_est) + + # Compact duplicate peak temperatures by averaging (diagnostic only) + if t_peaks: + uniq = {} + for t, val in zip(t_peaks, log_dem_estimates, strict=False): + uniq.setdefault(t, []).append(val) + t_peaks_uniq = np.array(sorted(uniq.keys())) + log_dem_uniq = np.array([np.mean(uniq[t]) for t in t_peaks_uniq]) + # Store raw estimated peaks for debugging/inspection if desired + self._raw_estimated_dem_peaks = (t_peaks_uniq, log_dem_uniq) + else: + self._raw_estimated_dem_peaks = (np.array([]), np.array([])) + + est_log_dem_on_grid = np.ones_like(self.logT) # March2026 + # est_log_dem_on_grid = np.zeros_like(self.logT) #Old + + # Return the initial first guessed DEM + # Store for later use by the solver + self._initial_log_dem = est_log_dem_on_grid + + return est_log_dem_on_grid + + def _prepare_spline_system(self): + """ + Prepare the spline-based DEM parameterization. + + This mirrors the IDL routine mp_prep and sets up all arrays needed by + the least-squares solver, including: + + - self.n_spl : number of spline knots + - self.spline_logT : knot positions (evenly spaced in log10(T)) + - self.spline_log_dem : initial values of log10(DEM) at each knot + - self.pm_matrix : response matrix multiplied by T * d(ln T) + - self.weights : 1.0 for non-zero intensity channels, 0.0 for zero-intensity channels + - self.abundances : all ones + + pm_matrix corresponds to: + pm[i, j] = R_i(T_j) * T_j * d(ln T) + + which appears in the forward model: + I_model_i = sum_j DEM(T_j) * pm[i, j] + """ + + # Number of channels + n_line = len(self._observed_intensities) + + # IDL: n_spl = min(n_line - 1, 7) - Make this a keyword in the class so use can tune it? + self.n_spl = min(max(n_line - 1, 1), 7) + + # Weights and abundances (IDL sets all =1) + self.weights = np.where(self._observed_intensities != 0.0, 1.0, 0.0) + self.abundances = np.ones(n_line, dtype=float) + + # pm_matrix = R(T) * T * dlnT (IDL line: emis * 10^t * along(10^dt)) + # units - DN/s/pix/cm^5 * K * dLnT * DEM == DN/s/PIX + T_linear = self.T.to_value(u.K) + + self.pm_matrix = (self._response_matrix * T_linear * self.dlnT).astype(float) + + # Knot positions are evenly spaced in logT (IDL spl_t) + self.spline_logT = np.linspace(self.logT.min(), self.logT.max(), self.n_spl) + + # Initial spline DEM values: sample from initial logDEM grid - IDL uses a cubic spline + # (IDL spline(est_t, est_dem, spl_t)) + interp_init = interp1d( + self.logT, + self._initial_log_dem, # IDL is flat logDEM = 1.0 + kind="linear", + bounds_error=False, + fill_value="extrapolate", + ) + + self.spline_log_dem = interp_init(self.spline_logT) + + def _build_lmfit_parameters(self): + """ + Build lmfit.Parameters object representing log10(DEM) at the spline knots. + IDL applies a lower bound of -20 only (no upper bound). + """ + + if not hasattr(self, "spline_log_dem"): + raise RuntimeError("Run _prepare_spline_system() first.") + + params = Parameters() + + # for i, val in enumerate(self.spline_log_dem): + # # Start each knot from the IDL flat initial guess + # # (log10 scaled DEM = 1.0), as xrt_dem_iter_solver line 233 does. + # params.add( + # f"knot_{i}", + # value=float(val), + # min=-20.0, + # max=None, + # vary=True, + # ) + for i in range(self.n_spl): + params.add( + f"knot_{i}", + value=float(self.spline_log_dem[i]), + min=-20.0, + vary=True, + ) + # #MayJOY + # params.add( + # f"knot_{i}", + # value=0.0, + # min=-20.0, + # max=None, + # vary=True + # ) + + return params + + def _reconstruct_dem_from_knots(self, params): + """ + Construct DEM(T) on self.logT using spline of log10(DEM) at knot positions. + Uses a natural cubic spline for n_spl >= 2, or a constant for n_spl == 1. + """ + knot_vals = np.array([params[f"knot_{i}"].value for i in range(self.n_spl)]) + + if self.n_spl == 1: + # Constant DEM across all temperatures + log_dem = np.full_like(self.logT, knot_vals[0]) + else: + cs = CubicSpline(self.spline_logT, knot_vals, bc_type="natural") + log_dem = cs(self.logT) + + log_dem = np.clip(log_dem, -300.0, 300.0) + # log_dem = np.clip(log_dem, -20.0, 300.0) #Debugging - too forced + dem = 10.0**log_dem + return dem + + def _residuals(self, params): + """ + Compute residuals for use by the least-squares optimizer. + + Residuals are computed as: + residual_i = (I_model_i - I_observed_i) / sigma_i + + where: + I_model = (pm_matrix @ DEM) * abundances + + Returns + ------- + ndarray + Residuals for each filter channel. + """ + + # 1. DEM(T) + dem = self._reconstruct_dem_from_knots(params) + + # 2. Modeled intensities (IDL: i_mod = (dem ## pm) * abunds) + i_mod = (self.pm_matrix @ dem) * self.abundances + + # 3. Observed (scaled) + y_scaled = self.intensities_scaled # i_obs / solv_factor + sigma_scaled = self.sigma_scaled_intensity_uncertainties + + # 4. Residuals = (i_mod - y_obs) * weights / sigma + residuals = (i_mod - y_scaled) * self.weights / sigma_scaled + + # # chi^2 history, mostly for debugging + chi2_val = np.sum(residuals**2) + # JOY March 2026 + if not hasattr(self, "_iteration_chi2"): + self._iteration_chi2 = [] + self._iteration_chi2.append(chi2_val) + + # chi2_val = np.sum(residuals**2) + # if not hasattr(self, "_iteration_chi2"): + # self._iteration_chi2 = [] + # self._iteration_chi2.append(chi2_val) + + return residuals + + def _solve_single_dem(self, observed_intensities_vals: np.ndarray): + """ + This method solves the DEM for one set of intensities only, without + Monte Carlo perturbation. + """ + + nf = self._normalization_factor + + # 1. scaled obs/uncertainties + self.intensities_scaled = observed_intensities_vals / nf + sigma_phys = self.intensity_uncertainties.to_value(u.DN / u.s) + self.sigma_scaled_intensity_uncertainties = sigma_phys / nf + + # 2. trivial nosolve case + if np.all(self.intensities_scaled == 0.0): + dem_model = np.zeros_like(self.logT) + dem_phys = dem_model * nf + modeled_intensities_phys = np.zeros_like(observed_intensities_vals) + return dem_phys, modeled_intensities_phys, 0.0, None + + # 3. initial guess (log10 DEM_model on grid) + init_log_dem = self._estimate_initial_dem() # flat ~ 1.0 in IDL + self._initial_log_dem = init_log_dem + self._iteration_chi2 = ( + [] + ) # reset chi2 history for this solve pass JOY March 2026 + + # 4. spline system using that initial guess + self._prepare_spline_system() + self.weights = np.where(observed_intensities_vals != 0.0, 1.0, 0.0) + params0 = self._build_lmfit_parameters() # values = initial_log_dem at knots + + # 5. run minimizer + result = minimize( + self._residuals, params0, max_nfev=self._max_iterations + ) # method='leastsq' + # ########### JOY-UPDATED_July16 + # # 5. run minimizer + # # IDL MPFIT's MAXITER caps LM *iterations*; lmfit's max_nfev caps + # # *function evaluations*. Each LM iteration costs ~(n_spl + 1) + # # evaluations (one per parameter for the Jacobian, plus the step), + # # so scale max_nfev to make max_iterations behave like MAXITER. + # result = minimize( + # self._residuals, + # params0, + # max_nfev=self._max_iterations * (self.n_spl + 1), + # ) # method='leastsq' (default), analogous to IDL MPFIT + # ############### + + # THIS is the critical part – use *result.params*, not params0 <<< + dem_model = self._reconstruct_dem_from_knots(result.params) # DEM_model(T) + dem_phys = dem_model * nf + + i_mod_scaled = (self.pm_matrix @ dem_model) * self.abundances + modeled_intensities_phys = i_mod_scaled * nf + + resid = self._residuals(result.params) + chisq = float(np.sum(resid**2)) + + return dem_phys, modeled_intensities_phys, chisq, result + + def solve(self): + """ + High-level DEM solver. + + Python analogue of IDL's xrt_dem_iterative2.pro: + + 1. Validate inputs. + 2. Build the logT grid and interpolate temperature responses. + 3. Solve ONE base DEM using the original (unperturbed) intensities. + 4. If Monte Carlo is requested (monte_carlo_runs > 0), perform N perturbed solves by adding Gaussian noise to the base intensities. + 5. Store all outputs on the instance for later analysis/plotting. + + After calling solve(), the following attributes are defined: + + Base solution + ------------- + logT : ndarray (n_T,) + log10 temperature grid [K]. + dem : ndarray (n_T,) + Best-fit DEM(T) in physical units [cm^-5 K^-1]. + chisq : float + Chi-square of the base fit (sum of squared residuals). + modeled_intensities : ndarray (n_channels,) + Best-fit modeled intensities in [DN s^-1 pix^-1]. + _base_fit_result : lmfit.MinimizerResult + Full lmfit result object for diagnostics. + + Monte Carlo products + -------------------- + mc_dem : ndarray (N+1, n_T) + DEM curves for base (row 0) and each Monte Carlo run (rows 1..N), + in physical units [cm^-5 K^-1]. + mc_chisq : ndarray (N+1,) + Chi-square values for base (index 0) and each MC run. + mc_base_obs : ndarray (N+1, n_channels) + Observed intensities [DN s^-1 pix^-1] for base + each MC run. + Row 0 = original observation; rows 1..N = perturbed. + mc_mod_obs : ndarray (N+1, n_channels) + Modeled intensities [DN s^-1 pix^-1] corresponding to mc_dem. + """ + + # Validate inputs (IDL: argument checks near top) + self.validate_inputs() + + # 1) Build logT grid and response matrix - IDL: regular logT grid + interpolated emissivities + self.create_logT_grid() + self._interpolate_responses_to_grid() + + # Base observed intensities in physical units [DN/s/pix] + base_obs_phys = np.asarray(self._observed_intensities, dtype=float) + + # 2) Solve BASE DEM (unperturbed intensities) Corresponds to the first call to xrt_dem_iter_nowidget in IDL. + + dem_base, mod_base, chisq_base, base_result = self._solve_single_dem( + observed_intensities_vals=base_obs_phys + ) + + # Store base solution + # self.logT_solution = self.logT.copy() # alias - REMOVE after testing + self.dem = dem_base # [cm^-5 K^-1] + self.chisq = chisq_base # chi-square + self.modeled_intensities = mod_base # [DN/s/pix] + self._base_fit_result = base_result + + # 3) Allocate Monte Carlo arrays (IDL: base_obs, dem_out, chisq, mod_obs) + + n_T = self.logT.size + n_ch = base_obs_phys.size + N = self.monte_carlo_runs + + self.mc_dem = np.zeros((N + 1, n_T), dtype=float) + self.mc_chisq = np.zeros((N + 1,), dtype=float) + self.mc_base_obs = np.zeros((N + 1, n_ch), dtype=float) + self.mc_mod_obs = np.zeros((N + 1, n_ch), dtype=float) + + # Row 0 = base solution (unperturbed) + self.mc_dem[0, :] = dem_base + self.mc_chisq[0] = chisq_base + self.mc_base_obs[0, :] = base_obs_phys + self.mc_mod_obs[0, :] = mod_base + + # 4 - Monte Carlo loop + if N > 0: + rng = np.random.default_rng() # like IDL's systime(1) seeding + + # Intensity uncertainties in physical units [DN/s/pix] + sigma_phys = self.intensity_uncertainties.to_value(u.DN / u.s) + + for ii in range(1, N + 1): + print(f" - Monte Carlo run {ii}/{N}", end="\r", flush=True) + + noise = rng.normal(loc=0.0, scale=sigma_phys, size=base_obs_phys.shape) + obs_pert = base_obs_phys + noise + obs_pert = np.maximum(obs_pert, 0.0) # IDL: >0 to avoid negatives + + dem_i, mod_i, chisq_i, _ = self._solve_single_dem( + observed_intensities_vals=obs_pert + ) + + # 4c) Store Monte Carlo results + self.mc_dem[ii, :] = dem_i + self.mc_chisq[ii] = chisq_i + self.mc_base_obs[ii, :] = obs_pert + self.mc_mod_obs[ii, :] = mod_i + + print(f" - Monte Carlo complete ({N} runs) ") # final line after loop + return self.dem + + def summary(self): + """ + Print a detailed, diagnostic summary of the DEM solver state. + + This provides: + - Input observation details + - Temperature grid configuration + - Response matrix status + - Spline system configuration + - Base DEM fit results + - Monte Carlo statistics (if available) + - Available plotting helpers + """ + + print("\n" + "=" * 76) + print(" XRTpy DEM Iterative — Solver Summary") + print("=" * 76) + + print("\nINPUT DATA") + print("-" * 70) + print(f" Filters: {self.filter_names}") + print(f" Number of channels: {len(self._observed_intensities)}") + if self._intensity_uncertainties is not None: + arr_str = np.array2string( + np.array(self._observed_intensities), + precision=3, + suppress_small=True, + separator=", ", + ) + msg = f"User-provided (DN/s):\n {arr_str}" + else: + msg = "Auto-estimated (3% of I, min=2 DN/s)" + + print(f" Intensity uncertainties - {msg}") + + print("\nTEMPERATURE GRID") + print("-" * 70) + if hasattr(self, "logT"): + print( + f" logT range: {self.logT[0]:.2f} to {self.logT[-1]:.2f} [log10 K]" + ) + print(f" Number of bins: {len(self.logT)}") + print(f" Grid spacing (dlogT): {self.dlogT:.3f}") + print(f" Natural log spacing: {self.dlnT:.4f}") + else: + print(" Grid has not been constructed (call solve()).") + + print("\nRESPONSE MATRIX") + print("-" * 70) + if hasattr(self, "_response_matrix"): + print( + f" Shape: {self._response_matrix.shape} (n_filters x n_T_bins)" + ) + print(f" Units: {self._response_unit}") + print(" Non-zero entries per filter:") + for i, fname in enumerate(self.filter_names): + n_nonzero = np.count_nonzero(self._response_matrix[i]) + peak_logT = self.logT[np.argmax(self._response_matrix[i])] + print( + f" {fname:<22} non-zero bins: {n_nonzero:>3d} peak at logT = {peak_logT:.2f}" + ) + else: + print(" Response matrix not constructed.") + + print("\nSOLVER CONFIGURATION") + print("-" * 70) + print(f" Normalization factor: {self.normalization_factor:.2e}") + print(f" Max iterations: {self.max_iterations}") + print(f" Monte Carlo runs: {self.monte_carlo_runs}") + if hasattr(self, "n_spl"): + print(f" Number of spline knots: {self.n_spl}") + print(f" Knot positions (logT): {np.round(self.spline_logT, 3)}") + else: + print(" Spline system not prepared yet.") + + print("\nBASE DEM SOLUTION") + print("-" * 70) + if hasattr(self, "dem"): + log_dem = np.log10(np.clip(self.dem, 1e-99, None)) + peak_idx = np.argmax(self.dem) + peak_logT = self.logT[peak_idx] + peak_dem = self.dem[peak_idx] + + n_dof = ( + max(1, len(self._observed_intensities) - self.n_spl) + if hasattr(self, "n_spl") + else len(self._observed_intensities) + ) + reduced_chi2 = self.chisq / n_dof + + print(f" DEM shape: {self.dem.shape}") + print( + f" Peak DEM: {peak_dem:.4e} at logT = {peak_logT:.2f} [cm⁻⁵ K⁻¹]" + ) + print( + f" log10(DEM) range: {log_dem.min():.2f} to {log_dem.max():.2f}" + ) + print(f" Chi-square: {self.chisq:.4e}") + print(f" Reduced chi-square: {reduced_chi2:.4e} (chi2 / {n_dof} dof)") + else: + print(" No DEM solution computed yet (call solve()).") + + print("\nMONTE CARLO ENSEMBLE") + print("-" * 70) + if hasattr(self, "mc_dem"): + N = self.mc_dem.shape[0] - 1 + print(f" MC realizations: {N}") + if N > 0: + mc_only = self.mc_dem[1:] # (N, nT) + np.log10(np.clip(mc_only, 1e-99, None)) + else: + print(" MC array allocated but N=0 (no Monte Carlo runs performed).") + else: + print(" No Monte Carlo results available.") + + print("\nPLOTTING HELPERS") + print("-" * 76) + print(" • solver.plot_dem() – Base DEM only") + print(" • solver.plot_dem_mc() – Base DEM + MC ensemble") + print("\n" + "=" * 76 + "\n") + + +XRTDEMIterative.plot_dem = dem_plotting.plot_dem +XRTDEMIterative.plot_dem_mc = dem_plotting.plot_dem_mc diff --git a/xrtpy/xrt_dem_iterative/test/.gitignore b/xrtpy/xrt_dem_iterative/test/.gitignore new file mode 100644 index 000000000..aa65eb788 --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/.gitignore @@ -0,0 +1,12 @@ + +# generated test outputs +*.png +*.mp4 +*.sav + +# generated DEM test outputs +xrtpy/xrt_dem_iterative/test/data/cases/**/plots_overlay*/ +xrtpy/xrt_dem_iterative/test/data/cases/**/*overlay*.png +xrtpy/xrt_dem_iterative/test/data/cases/**/*initial_dem*.png +xrtpy/xrt_dem_iterative/test/data/cases/**/*.mp4 +xrtpy/xrt_dem_iterative/test/data/cases/**/*.sav diff --git a/xrtpy/xrt_dem_iterative/test/__init__.py b/xrtpy/xrt_dem_iterative/test/__init__.py new file mode 100644 index 000000000..a57977f5c --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/__init__.py @@ -0,0 +1,8 @@ +""" +Test suite for xrtpy.xrt_dem_iterative and related DEM tools. + +This file marks the `tests/` directory as a Python package and allows +test modules to import from xrtpy and related packages cleanly. + +Use this space to define shared fixtures or helper utilities for tests if needed. +""" diff --git a/xrtpy/xrt_dem_iterative/test/conftest.py b/xrtpy/xrt_dem_iterative/test/conftest.py new file mode 100644 index 000000000..ce1c40806 --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/conftest.py @@ -0,0 +1,9 @@ +import sys +from pathlib import Path + +import matplotlib as mpl + +mpl.use("Agg") + +# Make utils_sav_io importable when pytest runs from the repo root +sys.path.insert(0, str(Path(__file__).parent)) diff --git a/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_112.txt b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_112.txt new file mode 100644 index 000000000..fd17a15a0 --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_112.txt @@ -0,0 +1,18 @@ +DEM index: 112 +10-Oct-2008 00:00:00 +Units: DN/s +Filter Intensity Error Percent +Al-mesh 4.35645e+06 67009.1 1.53816 +Al-poly 4.82556e+06 70471.7 1.46038 +C-poly 3.31920e+06 58619.0 1.76606 +Ti-poly 2.14168e+06 47286.4 2.20791 +Be-thin 1.08568e+06 33961.1 3.12809 +Be-med 215903. 6773.48 3.13728 +Al-med 93715.4 2878.83 3.07189 +Al-thick 8685.49 268.391 3.09011 +Be-thick 330.326 9.22694 2.79328 +Al-poly/Al-mesh 3.29523e+06 58410.6 1.77258 +Al-poly/Ti-poly 1.68334e+06 42037.7 2.49728 +Al-poly/Al-thick 8099.61 259.455 3.20330 +Al-poly/Be-thick 293.908 8.71760 2.96610 +C-poly/Ti-poly 1.25334e+06 36413.1 2.90529 diff --git a/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_14.txt b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_14.txt new file mode 100644 index 000000000..84b5802b7 --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_14.txt @@ -0,0 +1,18 @@ +DEM index: 14 +10-Oct-2008 00:00:00 +Units: DN/s +Filter Intensity Error Percent +Al-mesh 4949.20 142.990 2.88915 +Al-poly 5291.10 147.711 2.79169 +C-poly 3584.44 102.299 2.85397 +Ti-poly 2329.27 69.4233 2.98047 +Be-thin 1112.14 33.9430 3.05205 +Be-med 216.758 6.29030 2.90199 +Al-med 94.3998 2.94368 3.11831 +Al-thick 8.71282 0.380063 4.36211 +Be-thick 0.329589 0.0873814 26.5122 +Al-poly/Al-mesh 3559.41 101.951 2.86426 +Al-poly/Ti-poly 1769.88 50.8746 2.87446 +Al-poly/Al-thick 8.12673 0.367596 4.52330 +Al-poly/Be-thick 0.293511 0.0834596 28.4349 +C-poly/Ti-poly 1314.09 36.8096 2.80115 diff --git a/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_15.txt b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_15.txt new file mode 100644 index 000000000..ae32498e5 --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_15.txt @@ -0,0 +1,18 @@ +DEM index: 15 +10-Oct-2008 00:00:00 +Units: DN/s +Filter Intensity Error Percent +Al-mesh 7125.64 203.997 2.86285 +Al-poly 8228.90 261.454 3.17727 +C-poly 6206.10 190.757 3.07369 +Ti-poly 4310.72 133.715 3.10191 +Be-thin 2633.15 73.6869 2.79843 +Be-med 593.411 17.5156 2.95168 +Al-med 243.889 6.66166 2.73143 +Al-thick 23.1174 0.728584 3.15167 +Be-thick 1.48818 0.166503 11.1884 +Al-poly/Al-mesh 5791.74 184.470 3.18506 +Al-poly/Ti-poly 3590.57 102.384 2.85146 +Al-poly/Al-thick 21.6285 0.589805 2.72698 +Al-poly/Be-thick 1.33318 0.158474 11.8869 +C-poly/Ti-poly 2839.09 76.4378 2.69233 diff --git a/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_159.txt b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_159.txt new file mode 100644 index 000000000..327e6458a --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_159.txt @@ -0,0 +1,18 @@ +DEM index: 159 +10-Oct-2008 00:00:00 +Units: DN/s +Filter Intensity Error Percent +Al-mesh 12168.8 383.357 3.15032 +Al-poly 14160.2 412.622 2.91396 +C-poly 11054.7 301.769 2.72978 +Ti-poly 8086.46 216.947 2.68285 +Be-thin 5438.57 149.700 2.75257 +Be-med 1621.40 48.7544 3.00693 +Al-med 654.170 18.3656 2.80746 +Al-thick 72.3524 2.16314 2.98972 +Be-thick 6.14447 0.321695 5.23553 +Al-poly/Al-mesh 10006.6 287.494 2.87306 +Al-poly/Ti-poly 6866.51 200.357 2.91788 +Al-poly/Al-thick 67.8997 2.09747 3.08907 +Al-poly/Be-thick 5.55975 0.306779 5.51785 +C-poly/Ti-poly 5678.95 152.886 2.69214 diff --git a/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_65.txt b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_65.txt new file mode 100644 index 000000000..b8dcc5b4d --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_65.txt @@ -0,0 +1,18 @@ +DEM index: 65 +10-Oct-2008 00:00:00 +Units: DN/s +Filter Intensity Error Percent +Al-mesh 11529.3 308.010 2.67154 +Al-poly 13305.3 400.329 3.00879 +C-poly 10060.6 288.248 2.86512 +Ti-poly 7020.42 202.527 2.88482 +Be-thin 4331.36 134.025 3.09429 +Be-med 1007.78 27.1269 2.69175 +Al-med 416.894 12.3450 2.96118 +Al-thick 40.2357 1.13894 2.83067 +Be-thick 2.57338 0.213825 8.30914 +Al-poly/Al-mesh 9367.29 278.419 2.97224 +Al-poly/Ti-poly 5859.51 185.513 3.16602 +Al-poly/Al-thick 37.6581 1.10287 2.92863 +Al-poly/Be-thick 2.30740 0.203332 8.81216 +C-poly/Ti-poly 4653.81 138.778 2.98203 diff --git a/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_99.txt b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_99.txt new file mode 100644 index 000000000..b202a0753 --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/XRT_intensities_DEM_99.txt @@ -0,0 +1,18 @@ +DEM index: 99 +10-Oct-2008 00:00:00 +Units: DN/s +Filter Intensity Error Percent +Al-mesh 104483. 3034.96 2.90475 +Al-poly 119207. 3235.98 2.71458 +C-poly 87662.9 2403.84 2.74214 +Ti-poly 59826.3 1620.93 2.70939 +Be-thin 35197.6 1063.40 3.02122 +Be-med 8244.95 261.701 3.17408 +Al-med 3381.95 99.4482 2.94056 +Al-thick 338.260 9.33411 2.75945 +Be-thick 22.4599 0.718465 3.19888 +Al-poly/Al-mesh 83144.0 2342.73 2.81767 +Al-poly/Ti-poly 49162.3 1473.35 2.99692 +Al-poly/Al-thick 316.627 9.03881 2.85472 +Al-poly/Be-thick 20.1825 0.570277 2.82561 +C-poly/Ti-poly 38475.1 1110.40 2.88601 diff --git a/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/plot_idl_vs_xrtpy_dem.rtf b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/plot_idl_vs_xrtpy_dem.rtf new file mode 100644 index 000000000..6aefdb894 --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/data/synthetic_dem_data/plot_idl_vs_xrtpy_dem.rtf @@ -0,0 +1,8 @@ +{\rtf1\ansi\ansicpg1252\cocoartf2822 +\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +{\*\expandedcolortbl;;} +\margl1440\margr1440\vieww11520\viewh8400\viewkind0 +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 + +\f0\fs24 \cf0 Add info about the data in this this folder - synthetic_dem_data } \ No newline at end of file diff --git a/xrtpy/xrt_dem_iterative/test/older_testing_methods/__init__.py b/xrtpy/xrt_dem_iterative/test/older_testing_methods/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/xrtpy/xrt_dem_iterative/test/older_testing_methods/cases/case_20080104_110426/mc_intensities_20080104_110426_IDL.csv b/xrtpy/xrt_dem_iterative/test/older_testing_methods/cases/case_20080104_110426/mc_intensities_20080104_110426_IDL.csv new file mode 100644 index 000000000..feefdde5e --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/older_testing_methods/cases/case_20080104_110426/mc_intensities_20080104_110426_IDL.csv @@ -0,0 +1,101 @@ +run,Be-med,Al-mesh,Ti-poly,Al-poly,Be-thin +1,230.18749,173.14490,48.335204,91.423457,6.0615707 +2,252.02599,184.48511,44.942308,91.646777,6.4503697 +3,225.80054,185.20212,47.268215,94.292648,6.5309094 +4,226.32740,185.02488,45.182737,87.086904,5.3141584 +5,234.19204,190.34925,46.587577,94.440641,5.8928975 +6,246.02017,187.27234,50.881840,92.331356,5.6246261 +7,246.24560,182.24731,49.256743,93.409898,5.4328692 +8,237.36414,198.58896,41.997672,96.990167,6.0283140 +9,229.80146,181.82367,40.388113,87.578012,5.8997978 +10,237.98491,172.29282,46.400787,97.868546,5.0707957 +11,232.80834,175.51894,44.452282,90.714880,6.2648131 +12,233.59709,191.90281,48.300730,90.973854,6.7953242 +13,236.41314,184.95598,47.639193,90.902177,5.7433239 +14,235.14997,181.19401,44.867095,97.569041,5.6408795 +15,230.57663,189.06216,44.366460,87.322453,4.9733871 +16,238.08063,183.74442,44.447142,91.546333,5.7673465 +17,228.99689,178.63959,46.327214,98.547591,4.9752501 +18,229.24212,178.93956,44.854856,89.834105,5.4879068 +19,240.20591,185.84291,50.056391,93.004144,6.2772743 +20,242.20371,177.13037,46.794609,86.075988,5.5222204 +21,216.61045,159.92577,49.269462,94.394262,5.7072163 +22,245.50183,184.62661,48.632932,92.999801,6.3426631 +23,225.25611,170.60048,48.512406,90.403811,5.8353394 +24,237.21221,187.27215,46.341200,90.271926,6.2256546 +25,236.28693,187.36140,48.979795,88.249070,5.5389331 +26,225.23500,191.51090,44.578418,92.309093,5.5258202 +27,225.85806,181.79949,42.918186,92.350214,5.8835621 +28,238.00454,174.69112,48.213646,91.763493,5.9488809 +29,235.75906,173.44970,46.877727,87.632631,5.7992634 +30,223.95724,191.66667,47.429857,91.080966,5.7029033 +31,233.65801,189.20157,42.389866,89.324605,5.6759041 +32,235.15848,176.24285,43.471238,81.576064,4.7939382 +33,230.59054,184.24146,49.067877,93.444842,5.6903672 +34,235.38800,194.34078,52.475076,92.849708,5.7993158 +35,231.31504,181.54863,45.390821,84.365918,6.4718691 +36,236.81086,193.33176,42.395860,91.130228,5.7060422 +37,228.31245,176.16333,44.542549,88.451802,6.8363767 +38,221.37210,179.55739,47.292329,87.667414,6.8320638 +39,243.38632,175.30161,48.051430,96.656920,5.2058257 +40,225.05277,196.18956,43.854549,90.387745,5.6850297 +41,247.99653,184.99473,44.840851,88.703546,5.5500112 +42,228.62444,195.30149,42.517959,89.512895,6.5078948 +43,226.87371,175.27968,41.847745,99.347314,5.7892404 +44,247.74779,186.11111,48.629807,90.414238,5.4531976 +45,232.48620,189.13237,44.677888,91.501587,6.0129054 +46,226.19356,177.29586,45.189956,85.274167,5.8750543 +47,233.84310,176.77487,44.719518,88.315826,5.7521243 +48,218.47615,190.95573,46.042644,88.870459,5.3690090 +49,234.47739,186.49801,46.630511,91.702826,6.0782011 +50,227.93541,182.58262,44.810495,92.670788,5.1005720 +51,237.74413,187.28896,46.619077,89.370103,6.4444561 +52,247.79582,182.94593,48.492166,88.938100,5.9888718 +53,241.91281,184.68797,42.444927,82.114249,6.3479786 +54,242.32430,182.05145,47.736906,98.793759,6.3435155 +55,229.55333,183.84679,43.783552,89.131790,6.0541686 +56,218.93337,178.49554,43.955261,92.870742,6.2478718 +57,227.24223,190.43280,47.186737,93.695702,5.6539584 +58,240.39333,188.36626,46.134282,96.785360,5.2951645 +59,242.10609,185.49604,41.868799,95.509112,5.2921199 +60,224.96050,192.97628,47.627337,89.877669,7.2125621 +61,232.58811,190.76571,43.699720,88.702146,5.3641786 +62,234.37084,167.23296,50.333599,91.551919,4.9933512 +63,234.76220,190.63927,47.123289,89.988553,5.8109678 +64,220.43668,185.50093,41.965543,92.884369,6.2356919 +65,230.64542,189.03404,46.653498,90.072864,6.0426008 +66,227.24724,183.66616,44.283254,94.498134,5.6631203 +67,233.49184,180.05578,46.873861,92.234403,5.8671365 +68,225.33510,179.42728,51.064686,87.482262,5.5175582 +69,226.23573,191.31503,42.956599,94.126855,6.2679784 +70,240.58887,182.42447,43.789225,86.030725,6.0371873 +71,244.91586,178.58386,46.177312,91.605666,5.5530254 +72,247.86686,162.21760,48.710499,98.016966,5.9915738 +73,246.68352,183.31958,46.061855,91.488979,5.0272916 +74,231.02831,185.35640,43.255841,87.737759,5.9805126 +75,222.73929,168.31943,44.497663,93.878451,5.9242671 +76,238.57992,175.67977,50.941041,87.531917,5.7610009 +77,227.43107,190.79656,46.912640,87.598868,5.9850514 +78,229.36828,189.16025,47.820397,91.336093,6.2265961 +79,263.07509,180.86314,43.837696,89.234398,5.9761855 +80,227.67298,180.40122,48.371510,88.215913,5.4512022 +81,228.20765,185.81527,48.464053,95.330845,5.1379925 +82,235.60523,171.86533,46.389853,84.709289,5.6908261 +83,236.18472,176.20350,45.096965,88.401110,6.4809253 +84,238.35302,194.98416,44.276267,88.809902,5.1140437 +85,242.00528,185.38903,45.785995,82.640221,5.5755735 +86,238.71659,176.65109,44.862023,92.298120,6.5189378 +87,244.18114,187.96210,47.238043,93.075537,6.0197057 +88,245.99240,177.23339,48.227428,89.924616,5.6501135 +89,233.40719,183.19494,47.676922,88.907763,5.7367661 +90,254.28057,181.04397,47.316148,93.125183,6.0920487 +91,236.39500,179.16700,43.988840,91.127848,5.9514012 +92,228.66134,174.44183,47.191756,90.084032,4.9704761 +93,228.13822,190.66027,46.626860,88.980077,5.2338064 +94,240.47008,186.53771,47.875695,91.578956,5.6452903 +95,235.24302,181.75700,45.293982,89.771549,5.9860996 +96,234.88053,180.18280,45.864611,90.686810,5.3509759 +97,226.60032,186.90884,46.026654,92.875764,4.8813002 +98,231.92801,190.16168,46.527664,92.551447,6.4088040 +99,234.02678,183.43643,47.301312,94.267529,5.6827695 +100,236.44848,173.98537,42.721055,89.274895,5.6763210 diff --git a/xrtpy/xrt_dem_iterative/test/older_testing_methods/idl_vs_xrtpy_dem.py b/xrtpy/xrt_dem_iterative/test/older_testing_methods/idl_vs_xrtpy_dem.py new file mode 100644 index 000000000..ee8499cdf --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/older_testing_methods/idl_vs_xrtpy_dem.py @@ -0,0 +1,880 @@ +# ruff: noqa: E402, FBT003 +import matplotlib.animation as animation +import matplotlib.pyplot as plt +import numpy as np +from utils_case_io import ( + case_dir, + load_idl_dem_sav, + read_mc_intensities_csv, + run_dem_for_mc_csv, +) + +# NOTE-User will need Python 3.11 to run + +# CASE_DIR = case_dir("case_20080104_110426") +# csv_path = CASE_DIR / "mc_intensities_20080104_110426_IDL.csv" +# idl = load_idl_dem_sav(CASE_DIR / "xrt_dem_output_20080104_110426_MCITER100.sav") +# mc = read_mc_intensities_csv(csv_path) + +# CASE_DIR = case_dir("case_20080104_110426") +# csv_path = CASE_DIR / "mc_intensities_20080104_110426_IDL.csv" +# idl = load_idl_dem_sav(CASE_DIR / "xrt_dem_output_20260107_124503_MCITER100.sav") +# mc = read_mc_intensities_csv(csv_path) + + +CASE_DIR = case_dir("case_20260317_225659") +csv_path = CASE_DIR / "mc_intensities_20210720 1604_IDL.csv" +idl = load_idl_dem_sav(CASE_DIR / "xrt_dem_output_202107201604_MCITER100.sav") +mc = read_mc_intensities_csv(csv_path) + + +print(mc.filters) +print(mc.mc_intensities.shape) # (N, n_filters) +print(mc.df.head()) + +observation_date = "2021-07-20T16:04" + +# out = run_dem_for_mc_csv( +# csv_path=csv_path, +# observation_date=observation_date, +# intensity_errors=[ 8.9914,6.6961,2.3677, 3.9784,0.4987]#None, # keep None to match IDL default behavior +# ) + +out = run_dem_for_mc_csv( + csv_path=csv_path, + observation_date=observation_date, + intensity_errors=[ + 11.1806, + 6.6540, + 12.0541, + 0.0063, + 0.5018, + 2.3537, + ], # None, # keep None to match IDL default behavior +) + +# import pdb; pdb.set_trace() + +print("filters:", out.filters) +print("logT shape:", out.logT.shape) +print("dem_runs shape:", out.dem_runs.shape) +print("modeled_runs shape:", out.modeled_runs.shape) +print("chisq_runs shape:", out.chisq_runs.shape) + +# import pdb; pdb.set_trace() + +print("\n\n-New Section-\n") +print("IDL logT:", idl.logT.shape) +print("IDL dem_base:", idl.dem_base.shape) +print("IDL dem_runs:", idl.dem_runs.shape, "n_runs=", idl.n_runs) + + +# import numpy as np +# import matplotlib.pyplot as plt +# from pathlib import Path + +# def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: +# return np.log10(np.maximum(dem, floor)) + +# # ------------------------- +# # Choose which 10 runs to plot +# # - Here: first 10 MC runs (Run 1..10) +# # Your arrays are: +# # idl.dem_runs: (100, 26) -> MC runs only +# # out.dem_runs: (100, 26) -> MC runs only +# # So run number i corresponds to index i-1 in these arrays. +# # ------------------------- +# runs_to_plot = list(range(0, 99)) # 1..10 + +# # Sanity checks +# assert idl.logT.shape == out.logT.shape +# assert np.allclose(idl.logT, out.logT, atol=1e-8), "IDL and Python logT grids differ!" +# logT = out.logT + +# # Extract the 10 runs from each side +# idl_10 = np.stack([idl.dem_runs[i - 1] for i in runs_to_plot], axis=0) # (10, 26) +# py_10 = np.stack([out.dem_runs[i - 1] for i in runs_to_plot], axis=0) # (10, 26) + +# log_idl_10 = _log10_dem(idl_10) +# log_py_10 = _log10_dem(py_10) + +# # Global y-limits across all 10 overlays (same scale for every plot) +# ymin = np.min([log_idl_10.min(), log_py_10.min()]) +# ymax = np.max([log_idl_10.max(), log_py_10.max()]) + +# # Optional padding for readability +# pad = 0.05 * (ymax - ymin) if ymax > ymin else 0.5 +# ymin -= pad +# ymax += pad + +# # Output folder +# plots_dir = CASE_DIR / "plots_overlay_first10" +# plots_dir.mkdir(parents=True, exist_ok=True) + +# for k, run in enumerate(runs_to_plot): +# fig, ax = plt.subplots(figsize=(8, 5)) + +# ax.plot(logT, log_idl_10[k], label=f"IDL run {run}", linewidth=2) +# ax.plot(logT, log_py_10[k], label=f"XRTpy run {run}", linewidth=2, linestyle="--") + +# ax.set_title(f"DEM Overlay (Run {run})") +# ax.set_xlabel("log T (K)") +# ax.set_ylabel("log10(DEM)") +# ax.set_ylim(ymin, ymax) +# ax.grid(True, alpha=0.3) +# ax.legend() + +# out_png = plots_dir / f"dem_overlay_run_{run:03d}.png" +# fig.tight_layout() +# fig.savefig(out_png, dpi=200) +# plt.close(fig) + + +# print(f"\n\nSaved overlay plots to: {plots_dir}\n\n") + + +# # ----------------------------------------------------------------------------- +# # SECTION 2: Make an MP4 movie (ffmpeg) +# # ----------------------------------------------------------------------------- +# # If Matplotlib can't find ffmpeg automatically, uncomment and set path: +# # import matplotlib as mpl +# # mpl.rcParams["animation.ffmpeg_path"] = "/opt/homebrew/bin/ffmpeg" + +# fig, ax = plt.subplots(figsize=(8, 5)) + +# line_idl, = ax.plot([], [], linewidth=2, label="IDL") +# line_py, = ax.plot([], [], linewidth=2, linestyle="--", label="XRTpy") + +# ax.set_xlim(float(logT.min()), float(logT.max())) +# ax.set_ylim(ymin, ymax) +# ax.set_xlabel("log T (K)") +# ax.set_ylabel("log10(DEM)") +# ax.grid(True, alpha=0.3) +# ax.legend() + +# def update(frame: int): +# run_idx = runs_to_plot[frame] +# ax.set_title(f"DEM Overlay (Run {run_idx + 1})") + +# line_idl.set_data(logT, log_idl[frame]) +# line_py.set_data(logT, log_py[frame]) + +# return line_idl, line_py + +# ani = animation.FuncAnimation( +# fig, +# update, +# frames=len(runs_to_plot), +# blit=True, +# ) + +# movie_path = CASE_DIR / "dem_overlay.mp4" + +# writer = animation.FFMpegWriter( +# fps=2, # change speed here +# metadata={"artist": "xrtpy"}, +# bitrate=1800, +# ) + +# ani.save(movie_path, writer=writer, dpi=200) +# plt.close(fig) + +# print(f"Movie saved to: {movie_path}") + + +def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(dem, floor)) + + +# ------------------------- +# Choose runs to plot +# Use 1-based run numbers for filenames/titles, but index arrays with run-1 +# ------------------------- +runs_to_plot = list(range(1, 101)) # 1..10 (matches your "first 10" folder) +# If you want all 100: +# runs_to_plot = list(range(1, 101)) # 1..100 + +# Sanity checks +assert idl.logT.shape == out.logT.shape +assert np.allclose(idl.logT, out.logT, atol=1e-8), "IDL and Python logT grids differ!" +logT = out.logT + +# Extract selected runs from each side +idl_sel = np.stack([idl.dem_runs[r - 1] for r in runs_to_plot], axis=0) # (n, 26) +py_sel = np.stack([out.dem_runs[r - 1] for r in runs_to_plot], axis=0) # (n, 26) + +log_idl_10 = _log10_dem(idl_sel) +log_py_10 = _log10_dem(py_sel) + +# Global y-limits across all selected overlays (same scale) +ymin = float(min(log_idl_10.min(), log_py_10.min())) +ymax = float(max(log_idl_10.max(), log_py_10.max())) +pad = 0.05 * (ymax - ymin) if ymax > ymin else 0.5 +ymin -= pad +ymax += pad + +# ------------------------- +# SECTION 1: Save overlay PNGs +# ------------------------- +plots_dir = CASE_DIR / "plots_overlay_first10" +plots_dir.mkdir(parents=True, exist_ok=True) + +for k, run in enumerate(runs_to_plot): + fig, ax = plt.subplots(figsize=(8, 5)) + + ax.plot(logT, log_idl_10[k], label=f"IDL run {run}", linewidth=2) + ax.plot(logT, log_py_10[k], label=f"XRTpy run {run}", linewidth=2, linestyle="--") + + ax.set_title(f"DEM Overlay (Run {run})") + ax.set_xlabel("log T (K)") + ax.set_ylabel("log10(DEM)") + ax.set_ylim(ymin, ymax) + ax.grid(True, alpha=0.3) + ax.legend() + + out_png = plots_dir / f"dem_overlay_run_{run:03d}.png" + fig.tight_layout() + fig.savefig(out_png, dpi=200) + plt.close(fig) + +print(f"Saved {len(runs_to_plot)} overlay plots to: {plots_dir}") + +# ------------------------- +# SECTION 2: Make MP4 movie via ffmpeg +# ------------------------- +# If Matplotlib can't find ffmpeg automatically, uncomment: +# import matplotlib as mpl +# mpl.rcParams["animation.ffmpeg_path"] = "/opt/homebrew/bin/ffmpeg" + +fig, ax = plt.subplots(figsize=(8, 5)) + +(line_idl,) = ax.plot([], [], linewidth=2, label="IDL") +(line_py,) = ax.plot([], [], linewidth=2, linestyle="--", label="XRTpy") + +ax.set_xlim(float(logT.min()), float(logT.max())) +ax.set_ylim(ymin, ymax) +ax.set_xlabel("log T (K)") +ax.set_ylabel("log10(DEM)") +ax.grid(True, alpha=0.3) +ax.legend() + + +def update(frame: int): + run = runs_to_plot[frame] + ax.set_title(f"DEM Overlay (Run {run})") + line_idl.set_data(logT, log_idl_10[frame]) + line_py.set_data(logT, log_py_10[frame]) + return (line_idl, line_py) + + +ani = animation.FuncAnimation( + fig, + update, + frames=len(runs_to_plot), + blit=False, # safer (avoids the blit init crash you saw) +) + +movie_path = plots_dir / "dem_overlay.mp4" + +writer = animation.FFMpegWriter( + fps=6.5, + metadata={"artist": "xrtpy"}, + bitrate=1800, +) + +ani.save(movie_path, writer=writer, dpi=200) +plt.close(fig) + +print(f"Movie saved to: {movie_path}") + +###### + +# ----------------------------------------------------------------------------- +# Professional PNGs (2 panels) that MATCH the movie style: +# - Top: log10(DEM) overlay (IDL vs XRTpy) +# - Bottom: Δ(log10 DEM) at major logT points with ±1σ error bars (across runs) +# Δ(log10 DEM) = log10(DEM_XRTpy) - log10(DEM_IDL) +# ----------------------------------------------------------------------------- + +import matplotlib.pyplot as plt +import numpy as np + + +def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(dem, floor)) + + +# runs_to_plot should be 1-based run numbers, e.g.: +# runs_to_plot = list(range(1, 11)) # first 10 +# runs_to_plot = list(range(1, 101)) # all 100 + +# logT should already exist, and log_idl_10/log_py_10 should correspond to runs_to_plot order +# If you need to rebuild them: +# idl_sel = np.stack([idl.dem_runs[r - 1] for r in runs_to_plot], axis=0) +# py_sel = np.stack([out.dem_runs[r - 1] for r in runs_to_plot], axis=0) +# log_idl_10 = _log10_dem(idl_sel) +# log_py_10 = _log10_dem(py_sel) + +# ------------------------- +# Major logT points (5.0, 5.5, ... 8.0) and nearest indices +# ------------------------- +major_logT = np.arange(5.0, 8.0 + 1e-9, 0.5) +major_idx = np.array([int(np.argmin(np.abs(logT - t))) for t in major_logT]) +major_x = logT[major_idx] # actual grid values used + +# ------------------------- +# Δ(log10 DEM) across ALL selected runs +# ------------------------- +delta_all = log_py_10 - log_idl_10 # (n_runs_selected, 26) +sigma_delta = np.std(delta_all, axis=0) # (26,) +sigma_major = sigma_delta[major_idx] # (n_major,) + +# y-limits for bottom panel based on all runs at major points (stable scaling) +dmin = float(np.min(delta_all[:, major_idx])) +dmax = float(np.max(delta_all[:, major_idx])) +dpad = 0.15 * (dmax - dmin) if dmax > dmin else 0.5 +dmin -= dpad +dmax += dpad + +# Optional "agreement band" shading (dex). Set to None to disable. +delta_band = None # e.g. 0.10 for ±0.10 dex band, or None to disable + +# ------------------------- +# SECTION 1: Save professional PNGs (2 panels) +# ------------------------- +plots_dir = CASE_DIR / "plots_overlay_professional" +plots_dir.mkdir(parents=True, exist_ok=True) + +for k, run in enumerate(runs_to_plot): + # run is 1-based; k is 0-based index into log_idl_10/log_py_10/delta_all + chisq = float(out.chisq_runs[run - 1]) if hasattr(out, "chisq_runs") else np.nan + + fig, (ax1, ax2) = plt.subplots( + 2, + 1, + figsize=(8.5, 7.0), + sharex=True, + gridspec_kw={"height_ratios": [3, 1]}, + constrained_layout=True, + ) + + # ---- Top: DEM overlay ---- + ax1.plot(logT, log_idl_10[k], linewidth=2.2, label="IDL") + ax1.plot(logT, log_py_10[k], linewidth=2.2, linestyle="--", label="XRTpy") + ax1.set_ylabel("log10(DEM)") + ax1.set_ylim(ymin, ymax) + ax1.grid(True, alpha=0.25) + ax1.legend(loc="best", frameon=True) + + # Annotate run + chisq + ax1.text( + 0.02, + 0.95, + f"Run: {run}\n$\\chi^2$: {chisq:.3g}", + transform=ax1.transAxes, + va="top", + ha="left", + bbox={"boxstyle": "round", "alpha": 0.15}, + ) + + # ---- Bottom: Δ(log10 DEM) at major points with ±1σ error bars ---- + ax2.axhline(0.0, linewidth=1.3, alpha=0.8) + + y_major = delta_all[k, major_idx] # Δ at major points for this run + + ax2.errorbar( + major_x, + y_major, + yerr=sigma_major, + fmt="o", + capsize=3, + elinewidth=1.0, + markersize=4.5, + color="black", + ) + + if delta_band is not None: + ax2.axhspan(-delta_band, +delta_band, alpha=0.12) + + ax2.set_xlabel("log T (K)") + ax2.set_ylabel("Δ log10(DEM)\n(XRTpy − IDL)") + ax2.set_ylim(dmin, dmax) + ax2.grid(True, alpha=0.25) + + fig.suptitle("DEM Comparison: IDL vs XRTpy", y=1.02) + + out_png = plots_dir / f"dem_compare_run_{run:03d}.png" + fig.savefig(out_png, dpi=250) + plt.close(fig) + +print(f"Saved {len(runs_to_plot)} professional plots to: {plots_dir}") +# # ------------------------- +# # SECTION 2: Movie (MP4) with same professional layout +# # ------------------------- +# fig, (ax1, ax2) = plt.subplots( +# 2, 1, figsize=(8.5, 7.0), +# sharex=True, +# gridspec_kw={"height_ratios": [3, 1]}, +# constrained_layout=True, +# ) + +# # Pre-create artists (faster + cleaner) +# (line_idl,) = ax1.plot([], [], linewidth=2.2, label="IDL") +# (line_py,) = ax1.plot([], [], linewidth=2.2, linestyle="--", label="XRTpy") +# ax1.set_ylabel("log10(DEM)") +# ax1.set_ylim(ymin, ymax) +# ax1.grid(True, alpha=0.25) +# ax1.legend(loc="best", frameon=True) + +# # annotation text in top panel +# anno = ax1.text( +# 0.02, 0.95, "", +# transform=ax1.transAxes, +# va="top", ha="left", +# bbox=dict(boxstyle="round", alpha=0.15), +# ) + +# ax2.axhline(0.0, linewidth=1.5, alpha=0.7) +# (line_delta,) = ax2.plot([], [], linewidth=2.0) +# ax2.set_xlabel("log10(T [K])") +# ax2.set_ylabel("Δ dex") +# ax2.set_ylim(dmin, dmax) +# ax2.grid(True, alpha=0.25) + +# if delta_band is not None: +# ax2.axhspan(-delta_band, +delta_band, alpha=0.12) + +# ax2.set_xlim(float(logT.min()), float(logT.max())) +# fig.suptitle("DEM Comparison: IDL vs XRTpy", y=1.02) + +# def update(frame: int): +# run = runs_to_plot[frame] # 1-based +# chisq = float(out.chisq_runs[run - 1]) if hasattr(out, "chisq_runs") else np.nan + +# line_idl.set_data(logT, log_idl_10[frame]) +# line_py.set_data(logT, log_py_10[frame]) +# line_delta.set_data(logT, delta[frame]) + +# anno.set_text(f"Run: {run}\n$\\chi^2$: {chisq:.3g}") +# return (line_idl, line_py, line_delta, anno) + +# ani = animation.FuncAnimation(fig, update, frames=len(runs_to_plot), blit=False) + +# # Speed control +# fps = 15 # increase for faster video +# movie_path = plots_dir / "dem_compare_professional_v2.mp4" +# writer = animation.FFMpegWriter(fps=fps, metadata={"artist": "xrtpy"}, bitrate=2200) + +# ani.save(movie_path, writer=writer, dpi=250) +# plt.close(fig) + +# print(f"Movie saved to: {movie_path}") + +import matplotlib.animation as animation +import matplotlib.pyplot as plt +import numpy as np + +# ------------------------- +# Bottom-panel "major" logT points +# ------------------------- +major_logT = np.arange(5.0, 8.0 + 1e-9, 0.5) # 5.0, 5.5, ... 8.0 + +# For each major temp, pick the nearest index in the logT grid +major_idx = np.array([int(np.argmin(np.abs(logT - t))) for t in major_logT]) +major_x = logT[major_idx] # actual grid values used (closest to requested) + +# Compute delta(logDEM) for all frames/runs we are animating +# shape: (n_frames, 26) +delta_all = log_py_10 - log_idl_10 + +# Error bars: 1-sigma across frames at each temperature bin (professional context) +# shape: (26,) +sigma_delta = np.std(delta_all, axis=0) + +# Only at the major points: +sigma_major = sigma_delta[major_idx] + +# ------------------------- +# Figure with two subplots +# ------------------------- +fig = plt.figure(figsize=(9, 7)) +gs = fig.add_gridspec(nrows=2, ncols=1, height_ratios=[3, 1], hspace=0.12) + +ax_top = fig.add_subplot(gs[0, 0]) +ax_bot = fig.add_subplot(gs[1, 0], sharex=ax_top) + +# Top lines +(line_idl,) = ax_top.plot([], [], linewidth=2, label="IDL") +(line_py,) = ax_top.plot([], [], linewidth=2, linestyle="--", label="XRTpy") + +ax_top.set_xlim(float(logT.min()), float(logT.max())) +ax_top.set_ylim(ymin, ymax) +ax_top.set_ylabel("log10(DEM)") +ax_top.grid(True, alpha=0.3) +ax_top.legend(loc="best") + +# Bottom: delta points with error bars (we'll update their y-values each frame) +# We create an "errorbar container" once and then update the data each frame. +eb = ax_bot.errorbar( + major_x, + np.zeros_like(major_x), + yerr=sigma_major, + fmt="o", + capsize=3, + elinewidth=1, + color="Black", +) + +ax_bot.axhline(0.0, linewidth=1, alpha=0.8) +ax_bot.set_xlabel("log T (K)") +ax_bot.set_ylabel("Δ log10(DEM)\n(XRTpy - IDL)") +ax_bot.grid(True, alpha=0.3) + +# Set a reasonable fixed y-range for delta based on overall spread +# (prevents the bottom axis from jumping around frame-to-frame) +dmin = float(np.min(delta_all[:, major_idx])) +dmax = float(np.max(delta_all[:, major_idx])) +pad = 0.15 * (dmax - dmin) if dmax > dmin else 0.5 +ax_bot.set_ylim(dmin - pad, dmax + pad) + +# Prevent top subplot from repeating x tick labels +plt.setp(ax_top.get_xticklabels(), visible=False) + + +def update(frame: int): + run = runs_to_plot[frame] # run number (1-based in your setup) + + # Top panel + ax_top.set_title(f"DEM Overlay (Run {run})") + line_idl.set_data(logT, log_idl_10[frame]) + line_py.set_data(logT, log_py_10[frame]) + + # Bottom panel: delta at major temp points + y_major = delta_all[frame, major_idx] + + # Update errorbar artist: + # eb.lines[0] is the marker/line for the data points + eb.lines[0].set_data(major_x, y_major) + + # Return artists for blitting (we'll keep blit=False for robustness) + return (line_idl, line_py, eb.lines[0]) + + +ani = animation.FuncAnimation( + fig, + update, + frames=len(runs_to_plot), + blit=False, +) + +movie_path = plots_dir / "dem_overlay_first10_science.mp4" + +writer = animation.FFMpegWriter( + fps=7.5, # <-- make faster/slower here + metadata={"artist": "xrtpy"}, + bitrate=1800, +) + +ani.save(movie_path, writer=writer, dpi=200) +plt.close(fig) + +print(f"Science-style movie saved to: {movie_path}") +###### + +# Single plot: overlay ALL XRTpy DEM runs on one figure (log10 DEM vs logT) +# Assumes you already have: +# - out.dem_runs shape (n_runs, 26) (here n_runs=100) +# - out.logT shape (26,) + +import matplotlib.pyplot as plt +import numpy as np + + +def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(dem, floor)) + + +logT = out.logT +dem_runs = out.dem_runs # (100, 26) + +log_dem_runs = _log10_dem(dem_runs) + +fig, ax = plt.subplots(figsize=(9, 6)) + +# Plot every run (thin) +for i in range(log_dem_runs.shape[0]): + ax.plot(logT, log_dem_runs[i], linewidth=1.0, alpha=0.25) + + +ax.set_title("XRTpy DEM Monte Carlo Runs (All Overlaid)") +ax.set_xlabel("log T (K)") +ax.set_ylabel("log10(DEM)") +ax.grid(True, alpha=0.3) +ax.legend(loc="best") + +# Save +out_png = CASE_DIR / "xrtpy_dem_all_runs_overlay.png" +fig.tight_layout() +fig.savefig(out_png, dpi=250) +plt.close(fig) + +print(f"Saved: {out_png}") +###### +# Single plot: overlay ALL IDL DEM runs (log10 DEM vs logT) +# Assumes: +# - idl.dem_runs shape (100, 26) +# - idl.logT shape (26,) + +import matplotlib.pyplot as plt +import numpy as np + + +def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(dem, floor)) + + +logT = idl.logT +dem_runs = idl.dem_runs # (100, 26) + +log_dem_runs = _log10_dem(dem_runs) + +fig, ax = plt.subplots(figsize=(9, 6)) + +# Plot every IDL run +for i in range(log_dem_runs.shape[0]): + ax.plot(logT, log_dem_runs[i], linewidth=1.0, alpha=0.35) + +ax.set_title("IDL DEM Monte Carlo Runs (All Overlaid)") +ax.set_xlabel("log T (K)") +ax.set_ylabel("log10(DEM)") +ax.grid(True, alpha=0.3) + +# Save +out_png = CASE_DIR / "idl_dem_all_runs_overlay.png" +fig.tight_layout() +fig.savefig(out_png, dpi=250) +plt.close(fig) + +print(f"Saved: {out_png}") +###### +import matplotlib.pyplot as plt +import numpy as np + + +def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(dem, floor)) + + +# Ensure grids match +assert np.allclose(idl.logT, out.logT, atol=1e-8), "logT grids differ!" +logT = idl.logT + +log_idl = _log10_dem(idl.dem_runs) # (100, 26) +log_xrt = _log10_dem(out.dem_runs) # (100, 26) + +fig, ax = plt.subplots(figsize=(9, 6)) + +# --- Plot IDL (orange) --- +for i in range(log_idl.shape[0]): + ax.plot(logT, log_idl[i], color="orange", linewidth=1.0, alpha=0.35) + +# --- Plot XRTpy (blue) --- +for i in range(log_xrt.shape[0]): + ax.plot(logT, log_xrt[i], color="blue", linewidth=1.0, alpha=0.35) + +ax.set_title("DEM Monte Carlo Runs: IDL (Orange) vs XRTpy (Blue)") +ax.set_xlabel("log T (K)") +ax.set_ylabel("log10(DEM)") +ax.grid(True, alpha=0.3) + +# Create manual legend entries (so we don't get 200 legend lines) +from matplotlib.lines import Line2D + +legend_lines = [ + Line2D([0], [0], color="orange", lw=2, label="IDL"), + Line2D([0], [0], color="blue", lw=2, label="XRTpy"), +] +ax.legend(handles=legend_lines, loc="best") + +fig.tight_layout() + +out_png = CASE_DIR / "idl_vs_xrtpy_all_runs_overlay.png" +fig.savefig(out_png, dpi=250) +plt.close(fig) + +print(f"Saved: {out_png}") +###### + +import numpy as np + +print("\n--- INPUTS USED TO FEED XRTpy DEM ---") +print("observation_date:", observation_date) +print("csv_path:", csv_path) +print("filters (order):", mc.filters) +print("mc.mc_intensities shape:", mc.mc_intensities.shape) + +# What is the "base" / "nominal" row? +# In many pipelines row 0 is the nominal intensities, and rows 1..N-1 are MC perturbed. +base_idx = 0 + +base_intensities = np.asarray(mc.mc_intensities[base_idx], dtype=float) +print("\nBase (row 0) intensities used for XRTpy:") +for f, val in zip(mc.filters, base_intensities, strict=False): + print(f" {f:>12s}: {val:.6f} DN/s") + +print("\nFirst 3 MC rows (sanity):") +for i in range(3): + row = np.asarray(mc.mc_intensities[i], dtype=float) + print(f" row {i}: " + ", ".join([f"{v:.3f}" for v in row])) + + +import matplotlib.pyplot as plt +import numpy as np + + +def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(dem, floor)) + + +# Ensure temperature grids match +assert np.allclose(idl.logT, out.logT, atol=1e-8), "logT grids differ!" +logT = idl.logT + +# --- IDL initial DEM --- +idl_initial = idl.dem_base # from .sav file +log_idl_initial = _log10_dem(idl_initial) + +# --- XRTpy initial DEM --- +# If your class defines dem_base, use it. +# Otherwise fallback to first run. +if hasattr(out, "dem_base"): + xrt_initial = out.dem_base +else: + xrt_initial = out.dem_runs[0] + +log_xrt_initial = _log10_dem(xrt_initial) + +# ------------------------- +# Plot +# ------------------------- +fig, ax = plt.subplots(figsize=(8.5, 6)) + +ax.plot(logT, log_idl_initial, color="orange", linewidth=2.5, label="IDL (initial)") +ax.plot( + logT, + log_xrt_initial, + color="blue", + linewidth=2.5, + linestyle="--", + label="XRTpy (initial)", +) + +ax.set_title("Initial DEM Comparison: IDL vs XRTpy") +ax.set_xlabel("log T (K)") +ax.set_ylabel("log10(DEM)") +ax.grid(True, alpha=0.3) +ax.legend(loc="best") + +fig.tight_layout() + +out_png = CASE_DIR / "initial_dem_idl_vs_xrtpy.png" +fig.savefig(out_png, dpi=300) +plt.close(fig) + +print(f"Saved: {out_png}") +###### +# def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: +# dem = np.asarray(dem, dtype=float) +# return np.log10(np.maximum(dem, floor)) + +# def _run_metrics(log_idl: np.ndarray, log_py: np.ndarray, dem_floor: float = 1e-20): +# """ +# Returns (mean_abs_diff, max_abs_diff, n_used_bins) using a mask that ignores bins +# where both DEMs are ~zero (physically irrelevant tails). +# """ +# # mask in linear space +# idl_lin = 10 ** log_idl +# py_lin = 10 ** log_py +# mask = (idl_lin > dem_floor) | (py_lin > dem_floor) + +# if not np.any(mask): +# # nothing to compare (both essentially zero everywhere) +# return 0.0, 0.0, 0 + +# diff = log_py[mask] - log_idl[mask] +# mean_abs = float(np.mean(np.abs(diff))) +# max_abs = float(np.max(np.abs(diff))) +# return mean_abs, max_abs, int(mask.sum()) + +# # ---------------------------- +# # 101 DEM comparisons +# # ---------------------------- + +# # 1) IDL log-space arrays +# log_idl_base = _log10_dem(idl.dem_base) # (26,) +# log_idl_mc = _log10_dem(idl.dem_runs) # (100, 26) + +# # 2) Python: base DEM + MC DEMs +# # You already computed MC DEMs in `out.dem_runs` (100, 26). +# # Now compute base DEM ONCE using the base intensities you used in IDL. + +# base_intensities = np.array([197.823884, 172.347778, 43.104172, 86.083239, 5.400697], dtype=float) +# filters = ["Be-med", "Al-mesh", "Ti-poly", "Al-poly", "Be-thin"] +# observation_date = "2008-01-04T11:04:26" + +# responses = generate_temperature_responses(filters, observation_date) +# base_solver = XRTDEMIterative( +# observed_channel=filters, +# observed_intensities=base_intensities, +# temperature_responses=responses, +# monte_carlo_runs=0, # explicit base only +# ) +# base_solver.solve() +# py_dem_base = np.asarray(base_solver.dem, dtype=float) # (26,) +# log_py_base = _log10_dem(py_dem_base) + +# log_py_mc = _log10_dem(out.dem_runs) # (100, 26) + +# # 3) Sanity checks (shapes + grid) +# assert idl.logT.shape == base_solver.logT.shape == out.logT.shape +# assert np.allclose(idl.logT, out.logT, atol=1e-10) +# assert log_idl_mc.shape == log_py_mc.shape == (100, len(idl.logT)) + +# # 4) Metrics for base + 100 runs +# dem_floor = 1e-20 +# mean_tol = 0.35 # dex (start here, tighten later) +# max_tol = 0.90 # dex + +# results = [] + +# # Run 0: base +# m, M, nmask = _run_metrics(log_idl_base, log_py_base, dem_floor=dem_floor) +# results.append(("base", m, M, nmask)) + +# # Runs 1..100: MC +# for i in range(100): +# m, M, nmask = _run_metrics(log_idl_mc[i], log_py_mc[i], dem_floor=dem_floor) +# results.append((f"mc_{i+1:03d}", m, M, nmask)) + +# # 5) Summarize +# means = np.array([r[1] for r in results], dtype=float) +# maxes = np.array([r[2] for r in results], dtype=float) + +# print("\nPer-run DEM comparison (IDL vs XRTpy)") +# print(f" runs compared: {len(results)} (base + 100 MC)") +# print(f" dem_floor: {dem_floor:g}") +# print(f" mean|Δlog10DEM|: median={np.median(means):.3f} max={means.max():.3f}") +# print(f" max |Δlog10DEM|: median={np.median(maxes):.3f} max={maxes.max():.3f}") + +# # 6) Fail logic (strict all-pass; you can relax later) +# failed = [(name, m, M, nmask) for (name, m, M, nmask) in results if (m > mean_tol) or (M > max_tol)] +# if failed: +# print("\nWorst offenders (up to 10):") +# failed_sorted = sorted(failed, key=lambda x: (x[1], x[2]), reverse=True)[:10] +# for name, m, M, nmask in failed_sorted: +# print(f" {name}: mean={m:.3f} dex, max={M:.3f} dex, bins_used={nmask}") + +# raise AssertionError( +# f"{len(failed)}/{len(results)} runs exceeded tolerances " +# f"(mean_tol={mean_tol}, max_tol={max_tol})." +# ) diff --git a/xrtpy/xrt_dem_iterative/test/older_testing_methods/plots_results.py b/xrtpy/xrt_dem_iterative/test/older_testing_methods/plots_results.py new file mode 100644 index 000000000..cf96760f2 --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/older_testing_methods/plots_results.py @@ -0,0 +1,918 @@ +# ruff: noqa: E402, FBT003 +import matplotlib.animation as animation +import matplotlib.pyplot as plt +import numpy as np +from utils_case_io import ( + case_dir, + load_idl_dem_sav, + read_mc_intensities_csv, + run_dem_for_mc_csv, +) + +# NOTE-User will need Python 3.11 to run + +# CASE_DIR = case_dir("case_20210720_1604") +# csv_path = CASE_DIR / "mc_intensities_20210720_1604_IDL.csv" +# idl = load_idl_dem_sav(CASE_DIR / "xrt_dem_output_20210720_1604_MCITER100.sav") +# observation_date = "2021-07-20T16:04" + +CASE_DIR = case_dir("case_20260107_124503") +csv_path = CASE_DIR / "mc_intensities_20260107_124503_IDL.csv" +idl = load_idl_dem_sav(CASE_DIR / "xrt_dem_output_20260107_124503_MCITER100.sav") +observation_date = "2026-01-07T12:45:03" + + +mc = read_mc_intensities_csv(csv_path) + + +print(mc.filters) +print(mc.mc_intensities.shape) # (N, n_filters) +print(mc.df.head()) + + +out = run_dem_for_mc_csv( + csv_path=csv_path, + observation_date=observation_date, + intensity_errors=None, # keep None to match IDL default behavior +) + +# import pdb; pdb.set_trace() + +print("filters:", out.filters) +print("logT shape:", out.logT.shape) +print("dem_runs shape:", out.dem_runs.shape) +print("modeled_runs shape:", out.modeled_runs.shape) +print("chisq_runs shape:", out.chisq_runs.shape) + +# import pdb; pdb.set_trace() + +print("\n\n-New Section-\n") +print("IDL logT:", idl.logT.shape) +print("IDL dem_base:", idl.dem_base.shape) +print("IDL dem_runs:", idl.dem_runs.shape, "n_runs=", idl.n_runs) + + +# import numpy as np +# import matplotlib.pyplot as plt +# from pathlib import Path + +# def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: +# return np.log10(np.maximum(dem, floor)) + +# # ------------------------- +# # Choose which 10 runs to plot +# # - Here: first 10 MC runs (Run 1..10) +# # Your arrays are: +# # idl.dem_runs: (100, 26) -> MC runs only +# # out.dem_runs: (100, 26) -> MC runs only +# # So run number i corresponds to index i-1 in these arrays. +# # ------------------------- +# runs_to_plot = list(range(0, 99)) # 1..10 + +# # Sanity checks +# assert idl.logT.shape == out.logT.shape +# assert np.allclose(idl.logT, out.logT, atol=1e-8), "IDL and Python logT grids differ!" +# logT = out.logT + +# # Extract the 10 runs from each side +# idl_10 = np.stack([idl.dem_runs[i - 1] for i in runs_to_plot], axis=0) # (10, 26) +# py_10 = np.stack([out.dem_runs[i - 1] for i in runs_to_plot], axis=0) # (10, 26) + +# log_idl_10 = _log10_dem(idl_10) +# log_py_10 = _log10_dem(py_10) + +# # Global y-limits across all 10 overlays (same scale for every plot) +# ymin = np.min([log_idl_10.min(), log_py_10.min()]) +# ymax = np.max([log_idl_10.max(), log_py_10.max()]) + +# # Optional padding for readability +# pad = 0.05 * (ymax - ymin) if ymax > ymin else 0.5 +# ymin -= pad +# ymax += pad + +# # Output folder +# plots_dir = CASE_DIR / "plots_overlay_first10" +# plots_dir.mkdir(parents=True, exist_ok=True) + +# for k, run in enumerate(runs_to_plot): +# fig, ax = plt.subplots(figsize=(8, 5)) + +# ax.plot(logT, log_idl_10[k], label=f"IDL run {run}", linewidth=2) +# ax.plot(logT, log_py_10[k], label=f"XRTpy run {run}", linewidth=2, linestyle="--") + +# ax.set_title(f"DEM Overlay (Run {run})") +# ax.set_xlabel("log T (K)") +# ax.set_ylabel("log10(DEM)") +# ax.set_ylim(ymin, ymax) +# ax.grid(True, alpha=0.3) +# ax.legend() + +# out_png = plots_dir / f"dem_overlay_run_{run:03d}.png" +# fig.tight_layout() +# fig.savefig(out_png, dpi=200) +# plt.close(fig) + + +# print(f"\n\nSaved overlay plots to: {plots_dir}\n\n") + + +# # ----------------------------------------------------------------------------- +# # SECTION 2: Make an MP4 movie (ffmpeg) +# # ----------------------------------------------------------------------------- +# # If Matplotlib can't find ffmpeg automatically, uncomment and set path: +# # import matplotlib as mpl +# # mpl.rcParams["animation.ffmpeg_path"] = "/opt/homebrew/bin/ffmpeg" + +# fig, ax = plt.subplots(figsize=(8, 5)) + +# line_idl, = ax.plot([], [], linewidth=2, label="IDL") +# line_py, = ax.plot([], [], linewidth=2, linestyle="--", label="XRTpy") + +# ax.set_xlim(float(logT.min()), float(logT.max())) +# ax.set_ylim(ymin, ymax) +# ax.set_xlabel("log T (K)") +# ax.set_ylabel("log10(DEM)") +# ax.grid(True, alpha=0.3) +# ax.legend() + +# def update(frame: int): +# run_idx = runs_to_plot[frame] +# ax.set_title(f"DEM Overlay (Run {run_idx + 1})") + +# line_idl.set_data(logT, log_idl[frame]) +# line_py.set_data(logT, log_py[frame]) + +# return line_idl, line_py + +# ani = animation.FuncAnimation( +# fig, +# update, +# frames=len(runs_to_plot), +# blit=True, +# ) + +# movie_path = CASE_DIR / "dem_overlay.mp4" + +# writer = animation.FFMpegWriter( +# fps=2, # change speed here +# metadata={"artist": "xrtpy"}, +# bitrate=1800, +# ) + +# ani.save(movie_path, writer=writer, dpi=200) +# plt.close(fig) + +# print(f"Movie saved to: {movie_path}") + + +def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(dem, floor)) + + +# ------------------------- +# Choose runs to plot +# Use 1-based run numbers for filenames/titles, but index arrays with run-1 +# ------------------------- +runs_to_plot = list(range(1, 101)) # 1..10 (matches your "first 10" folder) +# If you want all 100: +# runs_to_plot = list(range(1, 101)) # 1..100 + +# Sanity checks +assert idl.logT.shape == out.logT.shape +assert np.allclose(idl.logT, out.logT, atol=1e-8), "IDL and Python logT grids differ!" +logT = out.logT + +# Extract selected runs from each side +idl_sel = np.stack([idl.dem_runs[r - 1] for r in runs_to_plot], axis=0) # (n, 26) +py_sel = np.stack([out.dem_runs[r - 1] for r in runs_to_plot], axis=0) # (n, 26) + +log_idl_10 = _log10_dem(idl_sel) +log_py_10 = _log10_dem(py_sel) + +# Global y-limits across all selected overlays (same scale) +ymin = float(min(log_idl_10.min(), log_py_10.min())) +ymax = float(max(log_idl_10.max(), log_py_10.max())) +pad = 0.05 * (ymax - ymin) if ymax > ymin else 0.5 +ymin -= pad +ymax += pad + +# ------------------------- +# SECTION 1: Save overlay PNGs +# ------------------------- +plots_dir = CASE_DIR / "plots_overlay" +plots_dir.mkdir(parents=True, exist_ok=True) + +for k, run in enumerate(runs_to_plot): + fig, ax = plt.subplots(figsize=(8, 5)) + + ax.plot(logT, log_idl_10[k], label=f"IDL run {run}", linewidth=2) + ax.plot(logT, log_py_10[k], label=f"XRTpy run {run}", linewidth=2, linestyle="--") + + ax.set_title(f"DEM Overlay (Run {run})") + ax.set_xlabel("log T (K)") + ax.set_ylabel("log10(DEM)") + ax.set_ylim(ymin, ymax) + ax.grid(True, alpha=0.3) + ax.legend() + + out_png = plots_dir / f"dem_overlay_run_{run:03d}.png" + fig.tight_layout() + fig.savefig(out_png, dpi=200) + plt.close(fig) + +print(f"Saved {len(runs_to_plot)} overlay plots to: {plots_dir}") + +# ------------------------- +# SECTION 2: Make MP4 movie via ffmpeg +# ------------------------- +# If Matplotlib can't find ffmpeg automatically, uncomment: +# import matplotlib as mpl +# mpl.rcParams["animation.ffmpeg_path"] = "/opt/homebrew/bin/ffmpeg" + +fig, ax = plt.subplots(figsize=(8, 5)) + +(line_idl,) = ax.plot([], [], linewidth=2, label="IDL") +(line_py,) = ax.plot([], [], linewidth=2, linestyle="--", label="XRTpy") + +ax.set_xlim(float(logT.min()), float(logT.max())) +ax.set_ylim(ymin, ymax) +ax.set_xlabel("log T (K)") +ax.set_ylabel("log10(DEM)") +ax.grid(True, alpha=0.3) +ax.legend() + + +def update(frame: int): + run = runs_to_plot[frame] + ax.set_title(f"DEM Overlay (Run {run})") + line_idl.set_data(logT, log_idl_10[frame]) + line_py.set_data(logT, log_py_10[frame]) + return (line_idl, line_py) + + +ani = animation.FuncAnimation( + fig, + update, + frames=len(runs_to_plot), + blit=False, # safer (avoids the blit init crash you saw) +) + +movie_path = plots_dir / "dem_overlay.mp4" + +writer = animation.FFMpegWriter( + fps=6.5, + metadata={"artist": "xrtpy"}, + bitrate=1800, +) + +ani.save(movie_path, writer=writer, dpi=200) +plt.close(fig) + +print(f"Movie saved to: {movie_path}") + +###### + +# ----------------------------------------------------------------------------- +# Professional PNGs (2 panels) that MATCH the movie style: +# - Top: log10(DEM) overlay (IDL vs XRTpy) +# - Bottom: Δ(log10 DEM) at major logT points with ±1σ error bars (across runs) +# Δ(log10 DEM) = log10(DEM_XRTpy) - log10(DEM_IDL) +# ----------------------------------------------------------------------------- + +import matplotlib.pyplot as plt +import numpy as np + + +def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(dem, floor)) + + +# runs_to_plot should be 1-based run numbers, e.g.: +# runs_to_plot = list(range(1, 11)) # first 10 +# runs_to_plot = list(range(1, 101)) # all 100 + +# logT should already exist, and log_idl_10/log_py_10 should correspond to runs_to_plot order +# If you need to rebuild them: +# idl_sel = np.stack([idl.dem_runs[r - 1] for r in runs_to_plot], axis=0) +# py_sel = np.stack([out.dem_runs[r - 1] for r in runs_to_plot], axis=0) +# log_idl_10 = _log10_dem(idl_sel) +# log_py_10 = _log10_dem(py_sel) + +# ------------------------- +# Major logT points (5.0, 5.5, ... 8.0) and nearest indices +# ------------------------- +major_logT = np.arange(5.0, 8.0 + 1e-9, 0.5) +major_idx = np.array([int(np.argmin(np.abs(logT - t))) for t in major_logT]) +major_x = logT[major_idx] # actual grid values used + +# ------------------------- +# Δ(log10 DEM) across ALL selected runs +# ------------------------- +delta_all = log_py_10 - log_idl_10 # (n_runs_selected, 26) +sigma_delta = np.std(delta_all, axis=0) # (26,) +sigma_major = sigma_delta[major_idx] # (n_major,) + +# y-limits for bottom panel based on all runs at major points (stable scaling) +dmin = float(np.min(delta_all[:, major_idx])) +dmax = float(np.max(delta_all[:, major_idx])) +dpad = 0.15 * (dmax - dmin) if dmax > dmin else 0.5 +dmin -= dpad +dmax += dpad + +# Optional "agreement band" shading (dex). Set to None to disable. +delta_band = None # e.g. 0.10 for ±0.10 dex band, or None to disable + +# ------------------------- +# SECTION 1: Save professional PNGs (2 panels) +# ------------------------- +plots_dir = CASE_DIR / "plots_overlay_w_diff" +plots_dir.mkdir(parents=True, exist_ok=True) + +for k, run in enumerate(runs_to_plot): + # run is 1-based; k is 0-based index into log_idl_10/log_py_10/delta_all + chisq = float(out.chisq_runs[run - 1]) if hasattr(out, "chisq_runs") else np.nan + + fig, (ax1, ax2) = plt.subplots( + 2, + 1, + figsize=(8.5, 7.0), + sharex=True, + gridspec_kw={"height_ratios": [3, 1]}, + constrained_layout=True, + ) + + # ---- Top: DEM overlay ---- + ax1.plot(logT, log_idl_10[k], linewidth=2.2, label="IDL") + ax1.plot(logT, log_py_10[k], linewidth=2.2, linestyle="--", label="XRTpy") + ax1.set_ylabel("log10(DEM)") + ax1.set_ylim(ymin, ymax) + ax1.grid(True, alpha=0.25) + ax1.legend(loc="best", frameon=True) + + # Annotate run + chisq + ax1.text( + 0.02, + 0.95, + f"Run: {run}\n$\\chi^2$: {chisq:.3g}", + transform=ax1.transAxes, + va="top", + ha="left", + bbox={"boxstyle": "round", "alpha": 0.15}, + ) + + # ---- Bottom: Δ(log10 DEM) at major points with ±1σ error bars ---- + ax2.axhline(0.0, linewidth=1.3, alpha=0.8) + + y_major = delta_all[k, major_idx] # Δ at major points for this run + + ax2.errorbar( + major_x, + y_major, + yerr=sigma_major, + fmt="o", + capsize=3, + elinewidth=1.0, + markersize=4.5, + color="black", + ) + + if delta_band is not None: + ax2.axhspan(-delta_band, +delta_band, alpha=0.12) + + ax2.set_xlabel("log T (K)") + ax2.set_ylabel("Δ log10(DEM)\n(XRTpy − IDL)") + ax2.set_ylim(dmin, dmax) + ax2.grid(True, alpha=0.25) + + fig.suptitle("DEM Comparison: IDL vs XRTpy", y=1.02) + + out_png = plots_dir / f"dem_compare_run_{run:03d}.png" + fig.savefig(out_png, dpi=250) + plt.close(fig) + +print(f"Saved {len(runs_to_plot)} professional plots to: {plots_dir}") +# # ------------------------- +# # SECTION 2: Movie (MP4) with same professional layout +# # ------------------------- +# fig, (ax1, ax2) = plt.subplots( +# 2, 1, figsize=(8.5, 7.0), +# sharex=True, +# gridspec_kw={"height_ratios": [3, 1]}, +# constrained_layout=True, +# ) + +# # Pre-create artists (faster + cleaner) +# (line_idl,) = ax1.plot([], [], linewidth=2.2, label="IDL") +# (line_py,) = ax1.plot([], [], linewidth=2.2, linestyle="--", label="XRTpy") +# ax1.set_ylabel("log10(DEM)") +# ax1.set_ylim(ymin, ymax) +# ax1.grid(True, alpha=0.25) +# ax1.legend(loc="best", frameon=True) + +# # annotation text in top panel +# anno = ax1.text( +# 0.02, 0.95, "", +# transform=ax1.transAxes, +# va="top", ha="left", +# bbox=dict(boxstyle="round", alpha=0.15), +# ) + +# ax2.axhline(0.0, linewidth=1.5, alpha=0.7) +# (line_delta,) = ax2.plot([], [], linewidth=2.0) +# ax2.set_xlabel("log10(T [K])") +# ax2.set_ylabel("Δ dex") +# ax2.set_ylim(dmin, dmax) +# ax2.grid(True, alpha=0.25) + +# if delta_band is not None: +# ax2.axhspan(-delta_band, +delta_band, alpha=0.12) + +# ax2.set_xlim(float(logT.min()), float(logT.max())) +# fig.suptitle("DEM Comparison: IDL vs XRTpy", y=1.02) + +# def update(frame: int): +# run = runs_to_plot[frame] # 1-based +# chisq = float(out.chisq_runs[run - 1]) if hasattr(out, "chisq_runs") else np.nan + +# line_idl.set_data(logT, log_idl_10[frame]) +# line_py.set_data(logT, log_py_10[frame]) +# line_delta.set_data(logT, delta[frame]) + +# anno.set_text(f"Run: {run}\n$\\chi^2$: {chisq:.3g}") +# return (line_idl, line_py, line_delta, anno) + +# ani = animation.FuncAnimation(fig, update, frames=len(runs_to_plot), blit=False) + +# # Speed control +# fps = 15 # increase for faster video +# movie_path = plots_dir / "dem_compare_professional_v2.mp4" +# writer = animation.FFMpegWriter(fps=fps, metadata={"artist": "xrtpy"}, bitrate=2200) + +# ani.save(movie_path, writer=writer, dpi=250) +# plt.close(fig) + +# print(f"Movie saved to: {movie_path}") +import matplotlib.animation as animation +import matplotlib.pyplot as plt +import numpy as np +from matplotlib.collections import LineCollection + + +def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(dem, floor)) + + +# ------------------------- +# Inputs assumed to exist: +# logT (nT,) +# log_idl_10 (n_runs, nT) +# log_py_10 (n_runs, nT) +# runs_to_plot (list of run numbers, 1-based) +# ymin, ymax +# plots_dir (Path) +# ------------------------- + +# Major points: only keep ones inside the grid +major_logT = np.arange(5.0, 8.0 + 1e-9, 0.5) +major_logT = major_logT[ + (major_logT >= logT.min() - 1e-9) & (major_logT <= logT.max() + 1e-9) +] + +major_idx = np.array([int(np.argmin(np.abs(logT - t))) for t in major_logT]) +major_x = logT[major_idx] + +# Delta across frames +delta_all = log_py_10 - log_idl_10 # (n_frames, nT) +sigma_delta = np.std(delta_all, axis=0) # (nT,) +sigma_major = sigma_delta[major_idx] # (n_major,) + +# Stable y-range for bottom panel +dmin = float(np.min(delta_all[:, major_idx])) +dmax = float(np.max(delta_all[:, major_idx])) +pad = 0.15 * (dmax - dmin) if dmax > dmin else 0.5 +ax_bot_ylim = (dmin - pad, dmax + pad) + +# ------------------------- +# Figure + axes +# ------------------------- +fig = plt.figure(figsize=(9, 7)) +gs = fig.add_gridspec(nrows=2, ncols=1, height_ratios=[3, 1], hspace=0.12) + +ax_top = fig.add_subplot(gs[0, 0]) +ax_bot = fig.add_subplot(gs[1, 0], sharex=ax_top) + +# Top lines +(line_idl,) = ax_top.plot([], [], linewidth=2, label="IDL") +(line_py,) = ax_top.plot([], [], linewidth=2, linestyle="--", label="XRTpy") + +ax_top.set_xlim(float(logT.min()), float(logT.max())) +ax_top.set_ylim(ymin, ymax) +ax_top.set_ylabel("log10(DEM)") +ax_top.grid(True, alpha=0.3) +ax_top.legend(loc="best") + +# Bottom baseline +ax_bot.axhline(0.0, linewidth=1, alpha=0.8) +ax_bot.set_xlabel("log T (K)") +ax_bot.set_ylabel("Δ log10(DEM)\n(XRTpy - IDL)") +ax_bot.grid(True, alpha=0.3) +ax_bot.set_ylim(*ax_bot_ylim) + +plt.setp(ax_top.get_xticklabels(), visible=False) + +# ------------------------- +# Build animated "errorbar" artists manually +# ------------------------- +# Markers +(points_line,) = ax_bot.plot( + major_x, np.zeros_like(major_x), "o", markersize=5, color="black" +) + + +# Vertical error segments (LineCollection) +def make_vsegments(y_major): + # segments shape: (n_major, 2, 2) => [ [ [x,ylo],[x,yhi] ], ...] + ylo = y_major - sigma_major + yhi = y_major + sigma_major + segs = np.zeros((len(major_x), 2, 2), dtype=float) + segs[:, 0, 0] = major_x + segs[:, 1, 0] = major_x + segs[:, 0, 1] = ylo + segs[:, 1, 1] = yhi + return segs + + +# Caps (small horizontal lines at top/bottom) +cap_halfwidth = 0.03 # in logT units (adjust if you want) + + +def make_capsegs(y_major): + ylo = y_major - sigma_major + yhi = y_major + sigma_major + + segs = [] + for x, yl, yh in zip(major_x, ylo, yhi, strict=False): + segs.append([[x - cap_halfwidth, yl], [x + cap_halfwidth, yl]]) + segs.append([[x - cap_halfwidth, yh], [x + cap_halfwidth, yh]]) + return np.array(segs, dtype=float) + + +vlines = LineCollection( + make_vsegments(np.zeros_like(major_x)), colors="black", linewidths=1.0 +) +caps = LineCollection( + make_capsegs(np.zeros_like(major_x)), colors="black", linewidths=1.0 +) +ax_bot.add_collection(vlines) +ax_bot.add_collection(caps) + + +# ------------------------- +# Update function +# ------------------------- +def update(frame: int): + run = runs_to_plot[frame] # 1-based run label + + # Top + ax_top.set_title(f"DEM Overlay (Run {run})") + line_idl.set_data(logT, log_idl_10[frame]) + line_py.set_data(logT, log_py_10[frame]) + + # Bottom + y_major = delta_all[frame, major_idx] + points_line.set_data(major_x, y_major) + + vlines.set_segments(make_vsegments(y_major)) + caps.set_segments(make_capsegs(y_major)) + + return (line_idl, line_py, points_line, vlines, caps) + + +ani = animation.FuncAnimation( + fig, + update, + frames=len(runs_to_plot), + blit=False, # keep robust +) + +movie_path = plots_dir / "dem_overlay_with_diff.mp4" +writer = animation.FFMpegWriter( + fps=7.5, # increase -> faster + metadata={"artist": "xrtpy"}, + bitrate=1800, +) + +ani.save(movie_path, writer=writer, dpi=200) +plt.close(fig) + +print(f"Movie saved to: {movie_path}") +# import numpy as np +# import matplotlib.pyplot as plt +# import matplotlib.animation as animation + +# # ------------------------- +# # Bottom-panel "major" logT points +# # ------------------------- +# major_logT = np.arange(5.0, 8.0 + 1e-9, 0.5) # 5.0, 5.5, ... 8.0 + +# # For each major temp, pick the nearest index in the logT grid +# major_idx = np.array([int(np.argmin(np.abs(logT - t))) for t in major_logT]) +# major_x = logT[major_idx] # actual grid values used (closest to requested) + +# # Compute delta(logDEM) for all frames/runs we are animating +# # shape: (n_frames, 26) +# delta_all = log_py_10 - log_idl_10 + +# # Error bars: 1-sigma across frames at each temperature bin (professional context) +# # shape: (26,) +# sigma_delta = np.std(delta_all, axis=0) + +# # Only at the major points: +# sigma_major = sigma_delta[major_idx] + +# # ------------------------- +# # Figure with two subplots +# # ------------------------- +# fig = plt.figure(figsize=(9, 7)) +# gs = fig.add_gridspec(nrows=2, ncols=1, height_ratios=[3, 1], hspace=0.12) + +# ax_top = fig.add_subplot(gs[0, 0]) +# ax_bot = fig.add_subplot(gs[1, 0], sharex=ax_top) + +# # Top lines +# (line_idl,) = ax_top.plot([], [], linewidth=2, label="IDL") +# (line_py,) = ax_top.plot([], [], linewidth=2, linestyle="--", label="XRTpy") + +# ax_top.set_xlim(float(logT.min()), float(logT.max())) +# ax_top.set_ylim(ymin, ymax) +# ax_top.set_ylabel("log10(DEM)") +# ax_top.grid(True, alpha=0.3) +# ax_top.legend(loc="best") + +# # Bottom: delta points with error bars (we'll update their y-values each frame) +# # We create an "errorbar container" once and then update the data each frame. +# eb = ax_bot.errorbar( +# major_x, +# np.zeros_like(major_x), +# yerr=sigma_major, +# fmt="o", +# capsize=3, +# elinewidth=1, +# color='Black' +# ) + +# ax_bot.axhline(0.0, linewidth=1, alpha=0.8) +# ax_bot.set_xlabel("log T (K)") +# ax_bot.set_ylabel("Δ log10(DEM)\n(XRTpy - IDL)") +# ax_bot.grid(True, alpha=0.3) + +# # Set a reasonable fixed y-range for delta based on overall spread +# # (prevents the bottom axis from jumping around frame-to-frame) +# dmin = float(np.min(delta_all[:, major_idx])) +# dmax = float(np.max(delta_all[:, major_idx])) +# pad = 0.15 * (dmax - dmin) if dmax > dmin else 0.5 +# ax_bot.set_ylim(dmin - pad, dmax + pad) + +# # Prevent top subplot from repeating x tick labels +# plt.setp(ax_top.get_xticklabels(), visible=False) + +# def update(frame: int): +# run = runs_to_plot[frame] # run number (1-based in your setup) + +# # Top panel +# ax_top.set_title(f"DEM Overlay (Run {run})") +# line_idl.set_data(logT, log_idl_10[frame]) +# line_py.set_data(logT, log_py_10[frame]) + +# # Bottom panel: delta at major temp points +# y_major = delta_all[frame, major_idx] + +# # Update errorbar artist: +# # eb.lines[0] is the marker/line for the data points +# eb.lines[0].set_data(major_x, y_major) + +# # Return artists for blitting (we'll keep blit=False for robustness) +# return (line_idl, line_py, eb.lines[0]) + +# ani = animation.FuncAnimation( +# fig, +# update, +# frames=len(runs_to_plot), +# blit=False, +# ) + +# movie_path = plots_dir / "dem_overlay_with_diff.mp4" + +# writer = animation.FFMpegWriter( +# fps=7.5, # <-- make faster/slower here +# metadata={"artist": "xrtpy"}, +# bitrate=1800, +# ) + +# ani.save(movie_path, writer=writer, dpi=200) +# plt.close(fig) + +# print(f"Science-style movie saved to: {movie_path}") +###### + +# Single plot: overlay ALL XRTpy DEM runs on one figure (log10 DEM vs logT) +# Assumes you already have: +# - out.dem_runs shape (n_runs, 26) (here n_runs=100) +# - out.logT shape (26,) + +import matplotlib.pyplot as plt +import numpy as np + + +def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(dem, floor)) + + +logT = out.logT +dem_runs = out.dem_runs # (100, 26) + +log_dem_runs = _log10_dem(dem_runs) + +fig, ax = plt.subplots(figsize=(9, 6)) + +# Plot every run (thin) +for i in range(log_dem_runs.shape[0]): + ax.plot(logT, log_dem_runs[i], linewidth=1.0, alpha=0.25) + + +ax.set_title("XRTpy DEM Monte Carlo Runs (All Overlaid)") +ax.set_xlabel("log T (K)") +ax.set_ylabel("log10(DEM)") +ax.grid(True, alpha=0.3) +ax.legend(loc="best") + +# Save +out_png = CASE_DIR / "xrtpy_dem_all_runs_overlay.png" +fig.tight_layout() +fig.savefig(out_png, dpi=250) +plt.close(fig) + +print(f"Saved: {out_png}") +###### +# Single plot: overlay ALL IDL DEM runs (log10 DEM vs logT) +# Assumes: +# - idl.dem_runs shape (100, 26) +# - idl.logT shape (26,) + +import matplotlib.pyplot as plt +import numpy as np + + +def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(dem, floor)) + + +logT = idl.logT +dem_runs = idl.dem_runs # (100, 26) + +log_dem_runs = _log10_dem(dem_runs) + +fig, ax = plt.subplots(figsize=(9, 6)) + +# Plot every IDL run +for i in range(log_dem_runs.shape[0]): + ax.plot(logT, log_dem_runs[i], linewidth=1.0, alpha=0.35) + +ax.set_title("IDL DEM Monte Carlo Runs (All Overlaid)") +ax.set_xlabel("log T (K)") +ax.set_ylabel("log10(DEM)") +ax.grid(True, alpha=0.3) + +# Save +out_png = CASE_DIR / "idl_dem_all_runs_overlay.png" +fig.tight_layout() +fig.savefig(out_png, dpi=250) +plt.close(fig) + +print(f"Saved: {out_png}") +###### +import matplotlib.pyplot as plt +import numpy as np + + +def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(dem, floor)) + + +# Ensure grids match +assert np.allclose(idl.logT, out.logT, atol=1e-8), "logT grids differ!" +logT = idl.logT + +log_idl = _log10_dem(idl.dem_runs) # (100, 26) +log_xrt = _log10_dem(out.dem_runs) # (100, 26) + +fig, ax = plt.subplots(figsize=(9, 6)) + +# --- Plot IDL (orange) --- +for i in range(log_idl.shape[0]): + ax.plot(logT, log_idl[i], color="orange", linewidth=1.0, alpha=0.35) + +# --- Plot XRTpy (blue) --- +for i in range(log_xrt.shape[0]): + ax.plot(logT, log_xrt[i], color="blue", linewidth=1.0, alpha=0.35) + +ax.set_title("DEM Monte Carlo Runs: IDL (Orange) vs XRTpy (Blue)") +ax.set_xlabel("log T (K)") +ax.set_ylabel("log10(DEM)") +ax.grid(True, alpha=0.3) + +# Create manual legend entries (so we don't get 200 legend lines) +from matplotlib.lines import Line2D + +legend_lines = [ + Line2D([0], [0], color="orange", lw=2, label="IDL"), + Line2D([0], [0], color="blue", lw=2, label="XRTpy"), +] +ax.legend(handles=legend_lines, loc="best") + +fig.tight_layout() + +out_png = CASE_DIR / "idl_vs_xrtpy_all_runs_overlay.png" +fig.savefig(out_png, dpi=250) +plt.close(fig) + +print(f"Saved: {out_png}") +###### + +import numpy as np + +print("\n--- INPUTS USED TO FEED XRTpy DEM ---") +print("observation_date:", observation_date) +print("csv_path:", csv_path) +print("filters (order):", mc.filters) +print("mc.mc_intensities shape:", mc.mc_intensities.shape) + +# What is the "base" / "nominal" row? +# In many pipelines row 0 is the nominal intensities, and rows 1..N-1 are MC perturbed. +base_idx = 0 + +base_intensities = np.asarray(mc.mc_intensities[base_idx], dtype=float) +print("\nBase (row 0) intensities used for XRTpy:") +for f, val in zip(mc.filters, base_intensities, strict=False): + print(f" {f:>12s}: {val:.6f} DN/s") + +print("\nFirst 3 MC rows (sanity):") +for i in range(3): + row = np.asarray(mc.mc_intensities[i], dtype=float) + print(f" row {i}: " + ", ".join([f"{v:.3f}" for v in row])) + + +import matplotlib.pyplot as plt +import numpy as np + + +def _log10_dem(dem: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(dem, floor)) + + +# Ensure temperature grids match +assert np.allclose(idl.logT, out.logT, atol=1e-8), "logT grids differ!" +logT = idl.logT + +# --- IDL initial DEM --- +idl_initial = idl.dem_base # from .sav file +log_idl_initial = _log10_dem(idl_initial) + +# --- XRTpy initial DEM --- +# If your class defines dem_base, use it. +# Otherwise fallback to first run. +if hasattr(out, "dem_base"): + xrt_initial = out.dem_base +else: + xrt_initial = out.dem_runs[0] + +log_xrt_initial = _log10_dem(xrt_initial) + +# ------------------------- +# Plot +# ------------------------- +fig, ax = plt.subplots(figsize=(8.5, 6)) + +ax.plot(logT, log_idl_initial, color="orange", linewidth=2.5, label="IDL (initial)") +ax.plot( + logT, + log_xrt_initial, + color="blue", + linewidth=2.5, + linestyle="--", + label="XRTpy (initial)", +) + +ax.set_title("Initial DEM Comparison: IDL vs XRTpy") +ax.set_xlabel("log T (K)") +ax.set_ylabel("log10(DEM)") +ax.grid(True, alpha=0.3) +ax.legend(loc="best") + +fig.tight_layout() + +out_png = CASE_DIR / "initial_dem_idl_vs_xrtpy.png" +fig.savefig(out_png, dpi=300) +plt.close(fig) + +print(f"Saved: {out_png}") diff --git a/xrtpy/xrt_dem_iterative/test/older_testing_methods/utils_case_io.py b/xrtpy/xrt_dem_iterative/test/older_testing_methods/utils_case_io.py new file mode 100644 index 000000000..8e9c1e34c --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/older_testing_methods/utils_case_io.py @@ -0,0 +1,286 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd +from scipy.io import readsav + +# from xrtpy.response.tools import generate_temperature_responses +from xrtpy.response.tools.multi_filter_response import generate_temperature_responses +from xrtpy.xrt_dem_iterative import XRTDEMIterative + + +def case_dir(case_name: str) -> Path: + return Path(__file__).parent / "data" / "cases" / case_name + + +@dataclass(frozen=True) +class MonteCarloInputs: + df: pd.DataFrame + filters: list[str] + mc_intensities: np.ndarray # shape: (n_runs, n_filters) + + +def read_mc_intensities_csv(csv_path: str | Path) -> MonteCarloInputs: + """ + Read Monte Carlo intensities CSV exported from IDL. + + Expected format: + run,,,... + 1, ... + 2, ... + ... + + Returns: + MonteCarloInputs with: + df: DataFrame (run + filter columns) + filters: list of filter column names + mc_intensities: float ndarray of shape (n_runs, n_filters) + """ + csv_path = Path(csv_path) + if not csv_path.exists(): + raise FileNotFoundError(f"CSV not found: {csv_path}") + + df = pd.read_csv(csv_path) + + # basic structure checks + if df.shape[1] < 2: + raise ValueError( + f"CSV must have at least 2 columns (run + filters). Got columns: {list(df.columns)}" + ) + + # Ensure we have a 'run' column (IDL sometimes writes 'run' exactly, but be safe) + if "run" not in df.columns: + # try case-insensitive match + matches = [c for c in df.columns if c.strip().lower() == "run"] + if not matches: + raise ValueError( + f"CSV missing a 'run' column. Found columns: {list(df.columns)}" + ) + df = df.rename(columns={matches[0]: "run"}) + + # normalize run + df["run"] = pd.to_numeric(df["run"], errors="raise").astype(int) + + # Sort by run just in case, and reset index + df = df.sort_values("run").reset_index(drop=True) + + # Require runs to be 1..N (strict on purpose for testing) + expected = np.arange(1, len(df) + 1) + if not np.array_equal(df["run"].to_numpy(), expected): + raise ValueError( + f"Run column must be consecutive 1..N.\n" + f"Expected: {expected[:10]}...\n" + f"Got: {df['run'].to_numpy()[:10]}..." + ) + + # filter columns + filters = [c for c in df.columns if c != "run"] + if len(filters) == 0: + raise ValueError("No filter columns found (only 'run' present).") + + # Convert all filter columns to float + for c in filters: + df[c] = pd.to_numeric(df[c], errors="raise").astype(float) + + mc_intensities = df[filters].to_numpy(dtype=float) # (n_runs, n_filters) + + # No NaNs - but shouldn't have any + if not np.isfinite(mc_intensities).all(): + bad = np.where(~np.isfinite(mc_intensities)) + raise ValueError(f"Found non-finite intensity values at indices: {bad}") + + return MonteCarloInputs(df=df, filters=filters, mc_intensities=mc_intensities) + + +@dataclass(frozen=True) +class DemBatchResult: + filters: list[str] + logT: np.ndarray # (nT,) # noqa: N815 + dem_runs: np.ndarray # (n_runs, nT) + modeled_runs: np.ndarray # (n_runs, n_filters) + chisq_runs: np.ndarray # (n_runs,) + + +def run_dem_for_mc_csv( + csv_path: str | Path, + observation_date: str, + *, + intensity_uncertainties: np.ndarray | None = None, + minimum_bound_temperature: float = 5.5, + maximum_bound_temperature: float = 8.0, + logarithmic_temperature_step_size: float = 0.1, +) -> DemBatchResult: + """ + Run XRTpy DEM for each row in an IDL-exported MC intensity CSV. + + Notes: + - We run monte_carlo_runs=0 because the CSV already provides the perturbed intensities. + - This makes the Python/IDL comparison deterministic. + """ + mc = read_mc_intensities_csv(csv_path) + filters = mc.filters + mc_intensities = mc.mc_intensities # (n_runs, n_filters) + + # Generate temperature responses once + responses = generate_temperature_responses(filters, observation_date) + + # ---- Ensure intensity_uncertainties length matches number of filters ---- + n_filters = len(filters) + + if intensity_uncertainties is not None: + # allow scalar -> broadcast + if np.isscalar(intensity_uncertainties): + intensity_uncertainties = np.full( + n_filters, float(intensity_uncertainties), dtype=float + ) + else: + intensity_uncertainties = np.asarray(intensity_uncertainties, dtype=float) + if intensity_uncertainties.shape[0] != n_filters: + raise ValueError( + f"intensity_uncertainties length ({intensity_uncertainties.shape[0]}) must match " + f"number of filters ({n_filters}). Filters={filters}" + ) + + # Create a solver "template" once (we'll reuse and only replace intensities) + solver = XRTDEMIterative( + observed_channel=filters, + observed_intensities=mc_intensities[0], + temperature_responses=responses, + intensity_uncertainties=intensity_uncertainties, # optional; can be None - Make sure current xrtpy-dem has it set- NOTEFORJOY + monte_carlo_runs=0, + minimum_bound_temperature=minimum_bound_temperature, + maximum_bound_temperature=maximum_bound_temperature, + logarithmic_temperature_step_size=logarithmic_temperature_step_size, + ) + + # Build grid/response matrix once + solver.create_logT_grid() + solver._interpolate_responses_to_grid() + + n_runs, n_filters = mc_intensities.shape + nT = len(solver.logT) + + dem_runs = np.zeros((n_runs, nT), dtype=float) + modeled_runs = np.zeros((n_runs, n_filters), dtype=float) + chisq_runs = np.zeros((n_runs,), dtype=float) + + # Run each case deterministically + for i in range(n_runs): + dem_i, modeled_i, chi2_i, _ = solver._solve_single_dem( + observed_intensities_vals=mc_intensities[i] + ) + dem_runs[i] = dem_i + modeled_runs[i] = modeled_i + chisq_runs[i] = chi2_i + + return DemBatchResult( + filters=filters, + logT=np.array(solver.logT, dtype=float), + dem_runs=dem_runs, + modeled_runs=modeled_runs, + chisq_runs=chisq_runs, + ) + + +@dataclass(frozen=True) +class IDLDemResult: + logT: np.ndarray # (nT,) # noqa: N815 + dem_runs: ( + np.ndarray + ) # (n_runs, nT) (runs exclude base or include base depending on file) + dem_base: np.ndarray # (nT,) + n_runs: int + + +def _to_1d(x) -> np.ndarray: + x = np.array(x) + return x.ravel() + + +def _ensure_runs_by_T(dem: np.ndarray, logT: np.ndarray) -> np.ndarray: + """ + Return DEM shaped (n_runs, nT). + + Handles common IDL orientations: + - (nT, nRuns+1) + - (nRuns+1, nT) + - sometimes (nT, nRuns) etc. + + We infer nT from logT length. + """ + dem = np.array(dem) + nT = len(logT) + + if dem.ndim != 2: + raise ValueError(f"Expected 2D DEM array from IDL, got shape {dem.shape}") + + a, b = dem.shape + + # Case 1: columns are runs, rows are temperature + if a == nT: + # dem is (nT, nRunsX) -> transpose to (nRunsX, nT) + return dem.T + + # Case 2: rows are runs, columns are temperature + if b == nT: + # dem already (nRunsX, nT) + return dem + + raise ValueError( + f"Cannot infer DEM orientation: dem shape={dem.shape}, logT length={nT}" + ) + + +def load_idl_dem_sav(sav_path: str | Path) -> IDLDemResult: + """ + Load IDL .sav file produced by your IDL DEM script and return: + - logT (nT,) + - dem_base (nT,) + - dem_runs (n_runs, nT) (MC only, base removed) + """ + + data = readsav(str(sav_path), python_dict=True) + + # Be flexible about key names + # Your newer IDL saver used logT_out/dem_out; older might be logt/dem_mc/dem + logT_key_candidates = [ + "logT_out", + "logt_out", + "logt", + "logT", + "logT_mc", + "logT_idl", + ] + dem_key_candidates = ["dem_out", "dem_mc", "dem", "dem0"] + + logT = None + for k in logT_key_candidates: + if k in data: + logT = _to_1d(data[k]) + break + if logT is None: + raise KeyError(f"Could not find logT in .sav. Keys: {sorted(data.keys())}") + + dem = None + for k in dem_key_candidates: + if k in data: + dem = np.array(data[k]) + break + if dem is None: + raise KeyError(f"Could not find DEM in .sav. Keys: {sorted(data.keys())}") + + dem_runs_all = _ensure_runs_by_T(dem, logT) # (nRunsX, nT) + + # Convention: in IDL output, column/run 0 is usually the base DEM + dem_base = dem_runs_all[0].copy() + dem_mc = dem_runs_all[1:].copy() + + return IDLDemResult( + logT=logT, + dem_base=dem_base, + dem_runs=dem_mc, + n_runs=dem_mc.shape[0], + ) diff --git a/xrtpy/xrt_dem_iterative/test/plot_idl_vs_xrtpy_dem.py b/xrtpy/xrt_dem_iterative/test/plot_idl_vs_xrtpy_dem.py new file mode 100644 index 000000000..bc916c4bf --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/plot_idl_vs_xrtpy_dem.py @@ -0,0 +1,598 @@ +""" +IDL vs XRTpy DEM Comparison — Multi-Case Plot Script +====================================================== + +Usage +----- +Plot one specific .sav file: + python plot_idl_vs_xrtpy_dem.py data/validation/xrt_IDL_dem_20080104T1104_..._.sav + +Plot all .sav files in the validation directory: + python plot_idl_vs_xrtpy_dem.py --all + +Plot all and save as PNG (no interactive window): + python plot_idl_vs_xrtpy_dem.py --all --save + +The script reads filter names and intensities directly from the filename — +no hardcoding required. +""" + +import argparse +import sys +from pathlib import Path + +import matplotlib.gridspec as gridspec +import matplotlib.pyplot as plt +import numpy as np + +from xrtpy.response.tools import generate_temperature_responses +from xrtpy.xrt_dem_iterative import XRTDEMIterative +from xrtpy.xrt_dem_iterative.utils_sav_io import ( + IDLResult, + SavCase, + discover_cases, + load_idl_sav, + parse_sav_filename, +) + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- +MEAN_TOL = 0.20 +MAX_TOL = 0.50 +DEM_FLOOR = 1e10 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _log10_safe(arr: np.ndarray, floor: float = 1e-99) -> np.ndarray: + return np.log10(np.maximum(arr, floor)) + + +def _valid_mask(dem_idl: np.ndarray, dem_xrt: np.ndarray) -> np.ndarray: + return (dem_idl > DEM_FLOOR) | (dem_xrt > DEM_FLOOR) + + +def _bar_color(d: float) -> str: + a = abs(d) + if a <= MEAN_TOL: + return "#1D9E75" + if a <= MAX_TOL: + return "#BA7517" + return "#E24B4A" + + +# --------------------------------------------------------------------------- +# Core: solve one case +# --------------------------------------------------------------------------- + + +def solve_case(case: SavCase) -> tuple[IDLResult, XRTDEMIterative]: + print(f" Loading IDL .sav: {case.sav_path.name}") + idl = load_idl_sav(case.sav_path) + + print(f" Running XRTpy solver ({len(case.filters)} filters)...") + responses = generate_temperature_responses(case.filters, case.observation_date) + solver = XRTDEMIterative( + observed_channel=case.filters, + observed_intensities=case.intensities_array, + temperature_responses=responses, + monte_carlo_runs=0, + ) + solver.solve() + + return idl, solver + + +# --------------------------------------------------------------------------- +# Core: make the 3-panel comparison figure +# --------------------------------------------------------------------------- + + +def make_figure(case: SavCase, idl: IDLResult, solver: XRTDEMIterative) -> plt.Figure: + + logT_idl = idl.logT + log_dem_idl = _log10_safe(idl.dem) + + logT_xrt = solver.logT + log_dem_xrt = _log10_safe(solver.dem) + modeled = solver.modeled_intensities + chisq = solver.chisq + + mask = _valid_mask(idl.dem, solver.dem) + delta = log_dem_xrt - log_dem_idl + mean_diff = float(np.mean(np.abs(delta[mask]))) + max_diff = float(np.max(np.abs(delta[mask]))) + + log_ratio = np.log10(np.maximum(modeled / case.intensities_array, 1e-99)) + + # Print summary to terminal + print(f"\n {'─' * 55}") + print(f" Case: {case.label} ({case.observation_date})") + print(f" Filters: {case.filters}") + print(f" XRTpy χ²: {chisq:.2f}") + print(f" Mean |Δ|: {mean_diff:.4f} dex") + print(f" Max |Δ|: {max_diff:.4f} dex") + print(f" IDL peak: logT = {logT_idl[np.argmax(idl.dem)]:.2f}") + print(f" XRTpy peak: logT = {logT_xrt[np.argmax(solver.dem)]:.2f}") + print(" Per-filter log10(modeled/observed):") + for f, lr in zip(case.filters, log_ratio, strict=True): + flag = " ← !" if abs(lr) > 1.0 else "" + print(f" {f:<22} {lr:+.3f}{flag}") + print(f" {'─' * 55}") + + # ── Build figure ───────────────────────────────────────────────────── + fig = plt.figure(figsize=(11, 13)) + fig.suptitle( + f"IDL vs XRTpy DEM Comparison\n" + f"Case: {case.observation_date} Filters: {', '.join(case.filters)}", + fontsize=12, + fontweight="bold", + y=0.98, + ) + + gs = gridspec.GridSpec(3, 1, figure=fig, height_ratios=[3, 2, 2], hspace=0.48) + + # ── Panel 1: DEM curves ────────────────────────────────────────────── + ax1 = fig.add_subplot(gs[0]) + + ax1.step( + logT_idl, + log_dem_idl, + where="mid", + color="#BA7517", + linewidth=2.2, + label="IDL reference", + ) + ax1.step( + logT_xrt, + log_dem_xrt, + where="mid", + color="#185FA5", + linewidth=2.0, + linestyle="--", + label="XRTpy", + ) + + # Shade high-T region where secondary rise often lives + ax1.axvspan( + 7.3, 8.05, alpha=0.07, color="crimson", label="High-T region (>logT 7.3)" + ) + + # Mark spline knot positions + knot_logT = np.linspace(logT_xrt.min(), logT_xrt.max(), solver.n_spl) + for j, klt in enumerate(knot_logT): + ax1.axvline( + klt, + color="#3B8BD4", + linewidth=0.7, + alpha=0.5, + linestyle=":", + label=f"XRTpy spline knots (n={solver.n_spl})" if j == 0 else "_nolegend_", + ) + + ax1.set_xlabel("log₁₀ T [K]") + ax1.set_ylabel("log₁₀ DEM [cm⁻⁵ K⁻¹]") + ax1.set_title("DEM(T) comparison", fontsize=11) + ax1.set_xlim(5.4, 8.1) + ax1.legend(fontsize=9, loc="upper right") + ax1.grid(visible=True, alpha=0.25) + + ax2 = fig.add_subplot(gs[1]) + + bar_colors = [_bar_color(d) for d in delta] + ax2.bar(logT_xrt, delta, width=0.09, color=bar_colors, alpha=0.85) + + ax2.axhline(0, color="black", linewidth=1.0, alpha=0.35) + ax2.axhline( + +MEAN_TOL, + color="#1D9E75", + linewidth=1.0, + linestyle="--", + label=f"±{MEAN_TOL} dex (mean tol.)", + alpha=0.8, + ) + ax2.axhline(-MEAN_TOL, color="#1D9E75", linewidth=1.0, linestyle="--", alpha=0.8) + ax2.axhline( + +MAX_TOL, + color="#BA7517", + linewidth=1.0, + linestyle=":", + label=f"±{MAX_TOL} dex (max tol.)", + alpha=0.8, + ) + ax2.axhline(-MAX_TOL, color="#BA7517", linewidth=1.0, linestyle=":", alpha=0.8) + ax2.fill_between([5.4, 8.1], -MEAN_TOL, MEAN_TOL, alpha=0.06, color="#1D9E75") + + ax2.set_xlabel("log₁₀ T [K]") + ax2.set_ylabel("Δ log₁₀ DEM\n(XRTpy − IDL)") + ax2.set_xlim(5.4, 8.1) + ax2.set_title( + f"Per-bin difference mean|Δ|={mean_diff:.3f} dex max|Δ|={max_diff:.3f} dex", + fontsize=11, + ) + ax2.legend(fontsize=9, loc="upper left") + ax2.grid(visible=True, alpha=0.25, axis="y") + + # ── Panel 3: per-filter intensities ────────────────────────────────── + ax3 = fig.add_subplot(gs[2]) + + n_filters = len(case.filters) + x = np.arange(n_filters) + w = 0.35 + + ax3.bar( + x - w / 2, + case.intensities_array, + w, + label="Observed", + color="#185FA5", + alpha=0.85, + ) + ax3.bar(x + w / 2, modeled, w, label="XRTpy modeled", color="#85B7EB", alpha=0.85) + + for i, (obs, mod) in enumerate(zip(case.intensities_array, modeled, strict=True)): + lr = np.log10(max(mod / obs, 1e-99)) + color = "#E24B4A" if abs(lr) > 1.0 else "#185FA5" + ax3.text( + i + w / 2, + mod * 1.3, + f"{lr:+.2f}", + ha="center", + va="bottom", + fontsize=8.5, + color=color, + fontweight="bold", + ) + + ax3.set_yscale("log") + ax3.set_xticks(x) + ax3.set_xticklabels(case.filters, rotation=20, ha="right", fontsize=9) + ax3.set_ylabel("Intensity [DN/s/pix]") + ax3.set_title( + "Per-filter intensities — numbers show log₁₀(modeled / observed)", + fontsize=11, + ) + ax3.legend(fontsize=9) + ax3.grid(visible=True, alpha=0.25, axis="y") + ax3.set_xlim(-0.6, n_filters - 0.4) + + ax3.text( + 0.98, + 0.97, + f"XRTpy χ² = {chisq:.1f}", + transform=ax3.transAxes, + ha="right", + va="top", + fontsize=9, + color="#E24B4A", + bbox={ + "boxstyle": "round,pad=0.3", + "facecolor": "white", + "edgecolor": "#E24B4A", + "alpha": 0.8, + }, + ) + + return fig + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Plot IDL vs XRTpy DEM comparison for one or more .sav files." + ) + parser.add_argument( + "sav_files", + nargs="*", + help="Path(s) to specific .sav file(s) to plot.", + ) + parser.add_argument( + "--all", + action="store_true", + help="Plot all xrt_IDL_dem_*.sav files in data/validation/base.", + ) + parser.add_argument( + "--save", + action="store_true", + help="Save each plot as a PNG file instead of showing interactively.", + ) + args = parser.parse_args() + + # Collect cases to plot + cases: list[SavCase] = [] + + if args.all: + data_dir = Path(__file__).parent / "data" / "validation" / "base" + cases = discover_cases(data_dir) + if not cases: + print(f"No xrt_IDL_dem_*.sav files found in {data_dir}") + sys.exit(1) + print(f"Found {len(cases)} case(s) in {data_dir}") + elif args.sav_files: + for p in args.sav_files: + cases.append(parse_sav_filename(Path(p))) + else: + parser.print_help() + sys.exit(0) + + # Process each case + for i, case in enumerate(cases, 1): + print(f"\n[{i}/{len(cases)}] Case: {case.label}") + print(f" Date: {case.observation_date}") + print(f" Filters: {case.filters}") + print(f" I_obs: {[f'{v:.3f}' for v in case.intensities]}") + + try: + idl, solver = solve_case(case) + except Exception as exc: # noqa: BLE001 + print(f" ERROR: {exc}") + continue + + fig = make_figure(case, idl, solver) + + # if args.save: + # out_path = Path(f"dem_comparison_{case.label}.png") + # fig.savefig(out_path, dpi=180, bbox_inches="tight") + # print(f" Saved: {out_path}") + # plt.close(fig) + if args.save: + case_dir = Path("plots") / case.label + case_dir.mkdir(parents=True, exist_ok=True) + out_path = case_dir / "base_dem.png" + fig.savefig(out_path, dpi=180, bbox_inches="tight") + print(f" Saved: {out_path}") + plt.close(fig) + else: + plt.tight_layout() + plt.show() + + print("\nDone.") + + +if __name__ == "__main__": + main() + +# """ +# IDL vs XRTpy DEM Comparison Plot +# ================================= +# Run this script to generate the real comparison using your actual XRTpy solver +# and the IDL reference .sav file. + +# Usage: +# python plot_idl_vs_xrtpy_dem.py + +# Requires: +# - scipy, numpy, matplotlib +# - xrtpy installed +# - xrt_IDL_dem_output_20080104.sav in the same directory +# """ + +# from pathlib import Path + +# import matplotlib.gridspec as gridspec +# import matplotlib.pyplot as plt +# import numpy as np +# from scipy.io import readsav + +# from xrtpy.response.tools import generate_temperature_responses +# from xrtpy.xrt_dem_iterative import XRTDEMIterative + +# # --------------------------------------------------------------------------- +# # Config +# # --------------------------------------------------------------------------- +# DATA_DIR = Path(__file__).parent / "data" / "validation" + +# SAV_FILE = DATA_DIR / "xrt_dem_output_20071213T0401_NOMC.sav"#"xrt_IDL_dem_output_20080104.sav" + +# # FILTERS = ["Be-med", "Al-mesh", "Ti-poly", "Al-poly", "Be-thin"] +# # INTENSITIES = np.array([234.283365, 183.711876, 45.931438, 91.745329, 5.755926]) +# # OBSERVATION_DATE = "2008-01-04T11:04:26" + +# FILTERS= ["be-med","Be-thin","Al-poly", "Al-poly/Ti-poly","Ti-poly","Al-thick"] +# INTENSITIES = [603.875886,150.921435,2412.340960, 301.354389 ,603.100596,2.519851] +# OBSERVATION_DATE = "2007-12-13T04:01" + + +# DEM_FLOOR = 1e10 + +# MEAN_TOL = 0.20 +# MAX_TOL = 0.50 + +# # --------------------------------------------------------------------------- +# # Load IDL reference +# # --------------------------------------------------------------------------- +# print("Loading IDL .sav file...") +# data = readsav(str(SAV_FILE), python_dict=True) +# logT_idl = np.array(data["logt"]).ravel().astype(float) +# dem_idl = np.array(data["dem"]).ravel().astype(float) +# log_dem_idl = np.log10(np.maximum(dem_idl, 1e-99)) + +# # --------------------------------------------------------------------------- +# # Run XRTpy solver +# # --------------------------------------------------------------------------- +# print("Running XRTpy DEM solver...") +# responses = generate_temperature_responses(FILTERS, OBSERVATION_DATE) +# solver = XRTDEMIterative( +# observed_channel=FILTERS, +# observed_intensities=INTENSITIES, +# temperature_responses=responses, +# monte_carlo_runs=0, +# ) +# solver.solve() + +# logT_xrt = solver.logT +# dem_xrt = solver.dem +# log_dem_xrt = np.log10(np.maximum(dem_xrt, 1e-99)) +# modeled = solver.modeled_intensities +# chisq = solver.chisq + +# print(f" XRTpy chi-square = {chisq:.2f}") +# print(f" IDL peak logT = {logT_idl[np.argmax(dem_idl)]:.2f}") +# print(f" XRTpy peak logT = {logT_xrt[np.argmax(dem_xrt)]:.2f}") + +# # --------------------------------------------------------------------------- +# # Metrics +# # --------------------------------------------------------------------------- +# mask = (dem_idl > DEM_FLOOR) | (dem_xrt > DEM_FLOOR) +# delta = log_dem_xrt - log_dem_idl +# mean_diff = np.mean(np.abs(delta[mask])) +# max_diff = np.max(np.abs(delta[mask])) +# print(f" Mean |Δlog10(DEM)| = {mean_diff:.3f} dex") +# print(f" Max |Δlog10(DEM)| = {max_diff:.3f} dex") + +# log_ratio = np.log10(np.maximum(modeled / INTENSITIES, 1e-99)) +# print(f" Per-filter log10(modeled/observed):") +# for f, lr in zip(FILTERS, log_ratio): +# flag = " ← !" if abs(lr) > 1.0 else "" +# print(f" {f:<18} {lr:+.3f} dex{flag}") + +# # --------------------------------------------------------------------------- +# # Plot +# # --------------------------------------------------------------------------- +# fig = plt.figure(figsize=(11, 13)) +# fig.suptitle( +# f"IDL vs XRTpy DEM Comparison\n" +# f"Case: {OBSERVATION_DATE} Filters: {', '.join(FILTERS)}", +# fontsize=13, fontweight="bold", y=0.98 +# ) + +# gs = gridspec.GridSpec( +# 3, 1, +# figure=fig, +# height_ratios=[3, 2, 2], +# hspace=0.45 +# ) + +# # ── Panel 1: DEM curves ────────────────────────────────────────────────── +# ax1 = fig.add_subplot(gs[0]) + +# ax1.step(logT_idl, log_dem_idl, where="mid", +# color="#BA7517", linewidth=2.2, label="IDL reference") +# ax1.step(logT_xrt, log_dem_xrt, where="mid", +# color="#185FA5", linewidth=2.0, linestyle="--", label="XRTpy") + +# # Highlight the secondary rise region +# ax1.axvspan(7.3, 8.0, alpha=0.08, color="crimson", label="Missing high-T region") + +# # Mark knot positions +# knot_logT = np.linspace(logT_xrt.min(), logT_xrt.max(), solver.n_spl) +# for klt in knot_logT: +# ax1.axvline(klt, color="#3B8BD4", linewidth=0.7, alpha=0.5, linestyle=":") +# ax1.axvline(knot_logT[0], color="#3B8BD4", linewidth=0.7, alpha=0.5, +# linestyle=":", label=f"XRTpy spline knots (n={solver.n_spl})") + +# ax1.set_xlabel("log₁₀ T [K]") +# ax1.set_ylabel("log₁₀ DEM [cm⁻⁵ K⁻¹]") +# ax1.set_title("DEM(T) comparison", fontsize=11) +# ax1.set_xlim(5.4, 8.1) +# ax1.legend(fontsize=9, loc="upper right") +# ax1.grid(True, alpha=0.25) + +# # Annotate peak +# pk_idl = logT_idl[np.argmax(dem_idl)] +# pk_xrt = logT_xrt[np.argmax(dem_xrt)] +# ax1.annotate(f"IDL peak\nlogT={pk_idl:.2f}", +# xy=(pk_idl, log_dem_idl.max()), +# xytext=(pk_idl - 0.6, log_dem_idl.max() - 3), +# fontsize=8, color="#BA7517", +# arrowprops=dict(arrowstyle="->", color="#BA7517", lw=0.8)) +# ax1.annotate(f"XRTpy peak\nlogT={pk_xrt:.2f}", +# xy=(pk_xrt, log_dem_xrt.max()), +# xytext=(pk_xrt + 0.15, log_dem_xrt.max() - 3.5), +# fontsize=8, color="#185FA5", +# arrowprops=dict(arrowstyle="->", color="#185FA5", lw=0.8)) + +# # ── Panel 2: Delta per bin ─────────────────────────────────────────────── +# ax2 = fig.add_subplot(gs[1]) + +# bar_colors = [] +# for d in delta: +# a = abs(d) +# if a <= MEAN_TOL: +# bar_colors.append("#1D9E75") +# elif a <= MAX_TOL: +# bar_colors.append("#BA7517") +# else: +# bar_colors.append("#E24B4A") + +# ax2.bar(logT_xrt, delta, width=0.09, color=bar_colors, alpha=0.8) + +# ax2.axhline(0, color="black", linewidth=1.0, alpha=0.4) +# ax2.axhline(+MEAN_TOL, color="#1D9E75", linewidth=1.0, linestyle="--", +# label=f"±{MEAN_TOL} dex (mean tol.)", alpha=0.8) +# ax2.axhline(-MEAN_TOL, color="#1D9E75", linewidth=1.0, linestyle="--", alpha=0.8) +# ax2.axhline(+MAX_TOL, color="#BA7517", linewidth=1.0, linestyle=":", +# label=f"±{MAX_TOL} dex (max tol.)", alpha=0.8) +# ax2.axhline(-MAX_TOL, color="#BA7517", linewidth=1.0, linestyle=":", alpha=0.8) + +# ax2.fill_between([5.4, 8.1], -MEAN_TOL, MEAN_TOL, alpha=0.06, color="#1D9E75") + +# ax2.set_xlabel("log₁₀ T [K]") +# ax2.set_ylabel("Δ log₁₀ DEM\n(XRTpy − IDL)") +# ax2.set_xlim(5.4, 8.1) +# ax2.set_title( +# f"Per-bin difference mean|Δ|={mean_diff:.3f} dex max|Δ|={max_diff:.3f} dex", +# fontsize=11 +# ) +# ax2.legend(fontsize=9, loc="upper left") +# ax2.grid(True, alpha=0.25, axis="y") + +# # ── Panel 3: Per-filter intensities ───────────────────────────────────── +# ax3 = fig.add_subplot(gs[2]) + +# x = np.arange(len(FILTERS)) +# w = 0.35 + +# b1 = ax3.bar(x - w/2, INTENSITIES, w, label="Observed", +# color="#185FA5", alpha=0.85) +# b2 = ax3.bar(x + w/2, modeled, w, label="XRTpy modeled", +# color="#85B7EB", alpha=0.85) + +# # Annotate log-ratio on each modeled bar +# for i, (obs, mod) in enumerate(zip(INTENSITIES, modeled)): +# ratio = np.log10(max(mod / obs, 1e-99)) +# color = "#E24B4A" if abs(ratio) > 1.0 else "#3B8BD4" +# ax3.text( +# i + w/2, mod * 1.15, +# f"{ratio:+.2f}", +# ha="center", va="bottom", +# fontsize=8.5, color=color, fontweight="bold" +# ) + +# ax3.set_yscale("log") +# ax3.set_xticks(x) +# ax3.set_xticklabels(FILTERS) +# ax3.set_ylabel("Intensity [DN/s/pix]") +# ax3.set_title( +# "Per-filter intensities — numbers show log₁₀(modeled/observed)", +# fontsize=11 +# ) +# ax3.legend(fontsize=9) +# ax3.grid(True, alpha=0.25, axis="y") +# ax3.set_xlim(-0.5, len(FILTERS) - 0.5) + +# # Add chi-square annotation +# ax3.text( +# 0.98, 0.97, +# f"XRTpy χ² = {chisq:.1f}", +# transform=ax3.transAxes, +# ha="right", va="top", +# fontsize=9, color="#E24B4A", +# bbox=dict(boxstyle="round,pad=0.3", facecolor="white", edgecolor="#E24B4A", alpha=0.8) +# ) + +# # --------------------------------------------------------------------------- +# # Save +# # --------------------------------------------------------------------------- +# out_path = Path("dem_idl_vs_xrtpy_comparison.png") +# fig.savefig(out_path, dpi=180, bbox_inches="tight") +# print(f"\nPlot saved to: {out_path}") +# plt.show() diff --git a/xrtpy/xrt_dem_iterative/test/plot_mc_comparison.py b/xrtpy/xrt_dem_iterative/test/plot_mc_comparison.py new file mode 100644 index 000000000..c43e74bd9 --- /dev/null +++ b/xrtpy/xrt_dem_iterative/test/plot_mc_comparison.py @@ -0,0 +1,1014 @@ +# """ +# plot_mc_comparison.py +# ===================== +# Four-panel Monte Carlo comparison: IDL vs XRTpy DEM ensembles. + +# Usage +# ----- +# python plot_mc_comparison.py --case 20071213T0401 +# python plot_mc_comparison.py --all +# python plot_mc_comparison.py --all --save + +# The script expects: +# data/validation/base/ xrt_IDL_dem_